From faeae16b84328d0b571a158d3eb57b3c180e8f3f Mon Sep 17 00:00:00 2001 From: Pyuyi <204892921+VeraPyuyi@users.noreply.github.com> Date: Mon, 19 Oct 2026 10:58:16 +0800 Subject: [PATCH 01/47] docs: define personal Dot task event bridge and routing boundaries Record approved ChatGPT-only MCP Events queue, separately paired local clients, durable idempotency, and accepted-job no-fallback requirements. Text routing and background jobs are distinct; live Dot roundtrip is required before readiness. Documentation only; source/design review completed, baseline go test ./... passed in sandbox. --- .../specs/2026-10-19-dot-routing-design.md | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 docs/superpowers/specs/2026-10-19-dot-routing-design.md diff --git a/docs/superpowers/specs/2026-10-19-dot-routing-design.md b/docs/superpowers/specs/2026-10-19-dot-routing-design.md new file mode 100644 index 0000000..b7c273f --- /dev/null +++ b/docs/superpowers/specs/2026-10-19-dot-routing-design.md @@ -0,0 +1,38 @@ +# Personal Dot automatic routing + +Approved goal: personal Dot is an automatic target for both text answers in Magpie routing groups and durable background cloud tasks. The user currently connects Dot through ChatGPT only. Existing manual context handoff remains useful input, not the finished goal. + +## Supported transport to verify + +OpenAI's MCP Events guide explicitly supports dots using protocol 2026-07-28. The user connects the private MCP plugin and asks Dot to subscribe to a task queue. DotControl stores submissions, emits a signed event to ChatGPT's supplied HTTPS callback, and provides authenticated tools to claim/read/update/complete tasks. ChatGPT's inbound MCP requests use the private tunnel. No new public listener or Slack/Teams account is required. Event receipt is not proof of execution. Live subscription/tool roundtrip must pass before readiness is advertised. + +Source: https://developers.openai.com/plugins/build/mcp-events and https://developers.openai.com/plugins/deploy/connect-chatgpt . Follow OpenAI's current supported webhook subset, not all experimental event delivery modes. + +## Modes + +Text routing initially accepts only supported text requests. Preserve roles and session references; reject or skip tools/images/strict schema before enqueue, without discarding fields. Tool-using Codex/OpenCode compatibility requires a separate verified agent protocol capability. + +Background tasks return persistent job IDs, progress, results and cancellation intent. They target Dot's own execution; existing DotControl target-Codex delegation must never be relabeled personal Dot cloud execution. + +## Core invariants + +- Explicit local opt-in; a separately paired Magpie client credential grants only its task queue submit/read/cancel operations, never existing admin/MCP credentials or desktop/history access. +- Durable request ID plus content digest makes retries idempotent. A conflicting reuse fails. Event retry preserves event ID and never creates a replacement job. +- Atomic claim binds a token to one job; duplicate events do not re-execute. Lost claims/leases become outcome_unknown rather than automatically executing a second cloud task. +- Persist queue/outbox before notification; persist accepted results. Corrupt/unreadable state fails closed. Shared state changes across processes serialize and replace atomically. +- Pause/revoke/disable block reads, claims and writes; in-flight operations recheck authority before publishing. Secret material uses the OS vault or encryption keys there. +- Subscription identity binds authenticated principal, canonical filters, event and callback. Persist expiration and signing secrets for granted lifetime. +- Callback delivery validates public HTTPS destinations at connection time, blocks local/private ranges and redirects, verifies challenges, and uses Standard Webhooks HMAC. Bounded delivery deadlines/retries; 410/413 stop retry. Send job IDs and small metadata rather than full prompts. +- A verified local queue acceptance must propagate through trusted plugin-host IPC/core state, not an untrusted upstream header. After acceptance, no gateway candidate/account/mirror fallback may duplicate the task. Client timeouts preserve original job references; cancellation of HTTP waiting does not prove cloud cancellation. +- Magpie's holdWriter can send HTTP 200/SSE comments while still falling back. Those are not a commit barrier. Never manufacture model answer text to lock routing. +- Text wait default is 120 seconds, with SSE comments only as protocol keepalive; no fabricated token streaming/usage. MCP tools are short calls within the tunnel's 120-second budget; cloud work runs outside them. + +## Incremental delivery + +1. DotControl durable queue, scoped client API, event discovery/subscription/delivery, claim/progress/complete tools and onboarding/probe commands integrated into serve. +2. Real ChatGPT/Dot subscription plus nonce-response roundtrip, duplicate/timeout evidence and observed readiness. +3. Magpie text provider/routing integration with capability preflight and acceptance barrier; isolated fixtures before live use. +4. Durable background task entry and status/results, using selected context. +5. Verified tool-call protocol before advertising agent-model compatibility. + +Each slice must be usable/testable, with independent review, deterministic fixtures, and no real-account evidence invented from tests. Upstream Magpie declines external PRs, so deliver fork drafts; DotControl changes can be separate small PRs. No merge/release merely from fixture passing. From f51a7a351ab35e46c0a4edf1ee5b0e9dbefe9539 Mon Sep 17 00:00:00 2001 From: Pyuyi <204892921+VeraPyuyi@users.noreply.github.com> Date: Mon, 19 Oct 2026 11:01:54 +0800 Subject: [PATCH 02/47] docs: stage Dot event bridge and automatic routing implementation Plan queue, authenticated event/client surface, runtime integration, Magpie capability and accepted-job barriers, provider/background tasks, then real Dot probe. Existing baseline passed; implementation verification remains per-task. Approved execution continues without another confirmation step. --- .../plans/2026-10-19-dot-automatic-routing.md | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 docs/superpowers/plans/2026-10-19-dot-automatic-routing.md diff --git a/docs/superpowers/plans/2026-10-19-dot-automatic-routing.md b/docs/superpowers/plans/2026-10-19-dot-automatic-routing.md new file mode 100644 index 0000000..d25e6d7 --- /dev/null +++ b/docs/superpowers/plans/2026-10-19-dot-automatic-routing.md @@ -0,0 +1,41 @@ +# Dot Automatic Routing Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use subagent-driven-development to implement task-by-task with independent review. User approved execution; proceed without another plan-approval pause. + +**Goal:** Connect personal Dot in ChatGPT to Magpie answer routing and durable background tasks using authenticated MCP Events and result tools. + +**Architecture:** DotControl owns durable encrypted queue, per-client access and event subscriptions. Magpie calls a loopback bridge, filters unsupported requests before enqueue and stops fallback after a verified queue acceptance. Event delivery and every MCP call are short operations; Dot runs work independently and writes results back. + +**Tech Stack:** Existing Go modules, crypto/net/http/json standard library, existing flock/vault. No browser-cookie scraping or normal inference API substitution. + +**Spec:** ../specs/2026-10-19-dot-routing-design.md; shared research contract 2026-10-19-dot-routing-wire-contract.md (resolve exact names across both detailed plans before writing product code). + +## Global Constraints + +- Explicit opt-in, dedicated paired-client credential, private loopback Host/Origin checks. +- No task accepted unless durable queue write succeeds; same request ID and digest returns same job, changed content conflicts. +- Unknown cloud effects never auto-reclaimed/reexecuted; cancel intent does not mean stopped. +- Protocol 2026-07-28 MCP Events webhook only; official discovery events capability must actually be present. +- Event callback HTTPS public addresses revalidated at dial, no redirects; challenge and Standard Webhooks signatures; secret24–64bytes in vault/encrypted state. +- Accept state conveyed by trusted internal control path, no vendor header spoof. Unsupported request must have zero queue writes. +- 120-second text wait default; no fake content/token usage; real Dot roundtrip needed for ready. +- Fixture tests use isolated dirs/vaults, no real configs/chats/secrets/services. + +## Review Focus + +- Two independent processes enqueueing same request: one persistent job and outbox ID. +- Worker restarts after claim with uncertain effect: outcome_unknown, no automatic second claim. +- DNS change/redirect/private callback: blocked before any secret/data leaves. +- Plugin or local request cancelled after acceptance: routing cannot create another task at another candidate. +- Expired/revoked subscription with queued work: not ready, no data/tool access or silent replay. + +## Tasks and dependency graph + +1. Durable queue and policy primitives, exact contracts in DotControl detail plan. Test transaction/restart/encryption/malformed state and claim lifecycle first, implement minimum, run focused+race, commit and review. +2. Authenticated loopback client API plus events/MCP tools with per-operation gate. Test unavailable/disabled separation, server discovery exact wire, unsigned/private callback rejection and full fixture roundtrip; implement and review. +3. Runtime/CLI enable/pair/status/probe integration and event worker lifecycle. Default 13 tools unchanged; enabling bridge additive. Test actual serve handler stack with fake vault and callback, shutdown cleanup and pause/revoke; review and full DotControl suite. +4. Magpie capability preflight and accepted-job barrier. Trace actual provider/gateway/plugin contracts before exact APIs; tests must fail for every retry/candidate/account/mirror bypass, upstream headers cannot set acceptance; review. +5. Dot provider local bridge and background task entry/status. Use shared v1 contract, short HTTP operations, stable request IDs and results. Test full gateway stack plus fakebridge/Dot worker (not fake feature modules), session separation and protocol conversion. Review, compile/vet/scope appropriate full suites, create drafts. +6. Real Dot acceptance: setup instructions and randomized probe ready before any account action; verify discovery/subscription/challenge/delivery/claim/completion, no fakefixture-ready setter. If account step is blocked, retain runnable draft and state exact missing evidence. + +The two detailed plans name exact files/interfaces/tests. This umbrella plan prevents a queue-only deliverable being called finished routing. From d279deef7610ea7e20a5c8eec3f75d5ba70b7826 Mon Sep 17 00:00:00 2001 From: Pyuyi <204892921+VeraPyuyi@users.noreply.github.com> Date: Mon, 19 Oct 2026 11:06:12 +0800 Subject: [PATCH 03/47] feat(dotqueue): durable encrypted task queue with tests Implement internal/dotqueue Store API with AES-GCM persistence, flock serialization, idempotent submit/claim/complete, cancel intent, explicit Recover, and bounded validation. Tests: go test ./internal/dotqueue/ -count=1 (pass); run_checks dotqueue-race -race (pass); run_checks dotcontrol-full go test ./... (pass). --- docs/DOT_TASK_QUEUE.md | 42 ++ internal/dotqueue/crypto.go | 62 +++ internal/dotqueue/digest.go | 69 ++++ internal/dotqueue/errors.go | 43 +++ internal/dotqueue/files_unix.go | 42 ++ internal/dotqueue/files_windows.go | 65 ++++ internal/dotqueue/json_strict.go | 72 ++++ internal/dotqueue/state.go | 186 +++++++++ internal/dotqueue/store.go | 599 +++++++++++++++++++++++++++++ internal/dotqueue/store_test.go | 307 +++++++++++++++ internal/dotqueue/types.go | 72 ++++ internal/dotqueue/validate.go | 119 ++++++ 12 files changed, 1678 insertions(+) create mode 100644 docs/DOT_TASK_QUEUE.md create mode 100644 internal/dotqueue/crypto.go create mode 100644 internal/dotqueue/digest.go create mode 100644 internal/dotqueue/errors.go create mode 100644 internal/dotqueue/files_unix.go create mode 100644 internal/dotqueue/files_windows.go create mode 100644 internal/dotqueue/json_strict.go create mode 100644 internal/dotqueue/state.go create mode 100644 internal/dotqueue/store.go create mode 100644 internal/dotqueue/store_test.go create mode 100644 internal/dotqueue/types.go create mode 100644 internal/dotqueue/validate.go diff --git a/docs/DOT_TASK_QUEUE.md b/docs/DOT_TASK_QUEUE.md new file mode 100644 index 0000000..600f8ff --- /dev/null +++ b/docs/DOT_TASK_QUEUE.md @@ -0,0 +1,42 @@ +# Dot task queue (`internal/dotqueue`) + +Task 1 delivers a durable, encrypted filesystem queue for DotControl routing work. Network exposure, MCP tools, vault wiring, and runtime gates are intentionally out of scope here. + +## API surface + +- `Open(dir, key)` — absolute state directory and 32-byte AES-256-GCM key supplied by caller (never persisted). +- `Store.Initialize(enabled)` — explicit first-time setup; writes `dot_queue.initialized` and refuses silent re-init if state is missing after the marker exists. +- `Store.SetEnabled(bool)` — toggles whether sensitive mutations are accepted. +- `Submit`, `Get`, `ByRequest`, `Claim`, `Progress`, `Complete`, `Cancel`, `Recover`, `Pending` — semantics match `docs/superpowers/specs/2026-10-19-dot-routing-design.md` and the task brief. + +Exported structs use JSON snake_case field names. Public `Job` values omit claim tokens and other worker secrets. + +## Storage and concurrency + +- Private directory (`0700`) and files (`0600`) with Windows current-user DACL, matching `internal/historyaccess` patterns. +- Cross-process `flock` on `dot_queue.lock` (5s acquisition budget, 10ms retry interval). +- All queue state is held in a single encrypted blob `queue.state.dq` (`D` + version byte + nonce + AES-GCM ciphertext). Prompts, results, and claim tokens are not stored in plaintext on disk. +- Updates are read → validate → mutate → temp file → sync → atomic replace under the lock. + +## Lifecycle rules + +- **Idempotency:** same `client_id` + `request_id` + submission digest returns the same job; digest mismatch is `dot_queue_conflict`. +- **Capacity:** at most 100 retained jobs; overflow returns conflict (no eviction). +- **Claim:** `queued` → `claimed` with a 256-bit token; duplicate claim by the same worker returns the same token; another worker gets `dot_queue_claim_conflict`. +- **Cancel:** before claim → `cancelled`; after claim → `cancel_requested` (progress does not clear cancel intent). +- **Complete:** terminal and idempotent on identical result digest; mismatched terminal payloads conflict. +- **Recover:** explicit only (not on `Open`); moves in-flight worker states to `outcome_unknown` once per job after restart uncertainty. +- **Disabled:** sensitive operations return `dot_queue_disabled`; read APIs remain available. + +## Error codes + +Stable `dotqueue.Error.Code` values include: `disabled`, `conflict`, `notfound`, `corrupt`, `claim_conflict`, `unknown`. + +## Tests + +Package tests cover encryption round-trip, reopen persistence, idempotency, claim/complete conflicts, cancel/recover, disabled gating, corrupt/missing state, bounds, concurrency across two `Store` handles, and capacity limits. Run: + +```text +go test ./internal/dotqueue/ -count=1 +python run_checks.py dotqueue-race go test ./internal/dotqueue/ -race -count=1 +``` diff --git a/internal/dotqueue/crypto.go b/internal/dotqueue/crypto.go new file mode 100644 index 0000000..d4327ce --- /dev/null +++ b/internal/dotqueue/crypto.go @@ -0,0 +1,62 @@ +package dotqueue + +import ( + "crypto/aes" + "crypto/cipher" + "crypto/rand" + "io" +) + +const envelopeVersion = 1 + +func seal(key []byte, plaintext []byte) ([]byte, error) { + if len(key) != 32 { + return nil, errf("unknown") + } + block, err := aes.NewCipher(key) + if err != nil { + return nil, errf("unknown") + } + gcm, err := cipher.NewGCM(block) + if err != nil { + return nil, errf("unknown") + } + nonce := make([]byte, gcm.NonceSize()) + if _, err := io.ReadFull(rand.Reader, nonce); err != nil { + return nil, errf("unknown") + } + ciphertext := gcm.Seal(nil, nonce, plaintext, nil) + out := make([]byte, 2+len(nonce)+len(ciphertext)) + out[0] = 'D' + out[1] = envelopeVersion + copy(out[2:], nonce) + copy(out[2+len(nonce):], ciphertext) + return out, nil +} + +func openEnvelope(key []byte, data []byte) ([]byte, error) { + if len(key) != 32 { + return nil, errf("corrupt") + } + if len(data) < 3 || data[0] != 'D' || data[1] != envelopeVersion { + return nil, errf("corrupt") + } + block, err := aes.NewCipher(key) + if err != nil { + return nil, errf("corrupt") + } + gcm, err := cipher.NewGCM(block) + if err != nil { + return nil, errf("corrupt") + } + if len(data) < 2+gcm.NonceSize() { + return nil, errf("corrupt") + } + nonce := data[2 : 2+gcm.NonceSize()] + ciphertext := data[2+gcm.NonceSize():] + plain, err := gcm.Open(nil, nonce, ciphertext, nil) + if err != nil { + return nil, errf("corrupt") + } + return plain, nil +} diff --git a/internal/dotqueue/digest.go b/internal/dotqueue/digest.go new file mode 100644 index 0000000..f24f712 --- /dev/null +++ b/internal/dotqueue/digest.go @@ -0,0 +1,69 @@ +package dotqueue + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" +) + +type canonicalSubmission struct { + RequestID string `json:"request_id"` + Mode string `json:"mode"` + SessionID string `json:"session_id"` + Messages []canonicalMessage `json:"messages"` + Task string `json:"task"` +} + +type canonicalMessage struct { + Role string `json:"role"` + Content string `json:"content"` +} + +type canonicalResult struct { + Text string `json:"text"` + Artifacts []canonicalArtifact `json:"artifacts"` + ErrorCode string `json:"error_code"` + Cancelled bool `json:"cancelled"` +} + +type canonicalArtifact struct { + URL string `json:"url"` + Label string `json:"label"` +} + +func submissionDigest(sub Submission) (string, error) { + msgs := make([]canonicalMessage, len(sub.Messages)) + for i, m := range sub.Messages { + msgs[i] = canonicalMessage{Role: m.Role, Content: m.Content} + } + body, err := json.Marshal(canonicalSubmission{ + RequestID: sub.RequestID, + Mode: sub.Mode, + SessionID: sub.SessionID, + Messages: msgs, + Task: sub.Task, + }) + if err != nil { + return "", errf("unknown") + } + sum := sha256.Sum256(body) + return hex.EncodeToString(sum[:]), nil +} + +func resultDigest(res Result) (string, error) { + arts := make([]canonicalArtifact, len(res.Artifacts)) + for i, a := range res.Artifacts { + arts[i] = canonicalArtifact{URL: a.URL, Label: a.Label} + } + body, err := json.Marshal(canonicalResult{ + Text: res.Text, + Artifacts: arts, + ErrorCode: res.ErrorCode, + Cancelled: res.Cancelled, + }) + if err != nil { + return "", errf("unknown") + } + sum := sha256.Sum256(body) + return hex.EncodeToString(sum[:]), nil +} diff --git a/internal/dotqueue/errors.go b/internal/dotqueue/errors.go new file mode 100644 index 0000000..85288a5 --- /dev/null +++ b/internal/dotqueue/errors.go @@ -0,0 +1,43 @@ +package dotqueue + +type Error struct { + Code string +} + +func (e *Error) Error() string { + if e == nil || e.Code == "" { + return "dot_queue_unknown" + } + return "dot_queue_" + e.Code +} + +func errf(code string) error { + return &Error{Code: code} +} + +func asQueueError(err error) *Error { + if err == nil { + return nil + } + if qe, ok := err.(*Error); ok { + return qe + } + return &Error{Code: "unknown"} +} + +func wrapUnknown(err error) error { + if err == nil { + return nil + } + if _, ok := err.(*Error); ok { + return err + } + return errf("unknown") +} + +func errorCode(err error) string { + if qe := asQueueError(err); qe != nil { + return qe.Code + } + return "unknown" +} diff --git a/internal/dotqueue/files_unix.go b/internal/dotqueue/files_unix.go new file mode 100644 index 0000000..c8f18c0 --- /dev/null +++ b/internal/dotqueue/files_unix.go @@ -0,0 +1,42 @@ +//go:build !windows + +package dotqueue + +import ( + "os" + "syscall" +) + +func privatePath(path string, directory bool) error { + mode := os.FileMode(0600) + if directory { + mode = 0700 + } + return os.Chmod(path, mode) +} + +func rejectLink(info os.FileInfo) error { + if info.Mode()&os.ModeSymlink != 0 { + return errf("unknown") + } + return nil +} + +func rejectHardlink(f *os.File, info os.FileInfo) error { + stat, ok := info.Sys().(*syscall.Stat_t) + if !ok || stat.Nlink != 1 { + return errf("unknown") + } + return nil +} + +func replaceFile(from, to string) error { return os.Rename(from, to) } + +func syncDirectory(path string) error { + f, err := os.Open(path) + if err != nil { + return err + } + defer f.Close() + return f.Sync() +} diff --git a/internal/dotqueue/files_windows.go b/internal/dotqueue/files_windows.go new file mode 100644 index 0000000..c68a9f1 --- /dev/null +++ b/internal/dotqueue/files_windows.go @@ -0,0 +1,65 @@ +//go:build windows + +package dotqueue + +import ( + "os" + "syscall" + + "golang.org/x/sys/windows" +) + +func privatePath(path string, directory bool) error { + user, err := windows.GetCurrentProcessToken().GetTokenUser() + if err != nil { + return err + } + inherit := "" + if directory { + inherit = "OICI" + } + sd, err := windows.SecurityDescriptorFromString("D:P(A;" + inherit + ";FA;;;" + user.User.Sid.String() + ")") + if err != nil { + return err + } + acl, _, err := sd.DACL() + if err != nil { + return err + } + return windows.SetNamedSecurityInfo(path, windows.SE_FILE_OBJECT, windows.DACL_SECURITY_INFORMATION|windows.PROTECTED_DACL_SECURITY_INFORMATION, nil, nil, acl, nil) +} + +func rejectLink(info os.FileInfo) error { + if info.Mode()&os.ModeSymlink != 0 { + return errf("unknown") + } + if attr, ok := info.Sys().(*syscall.Win32FileAttributeData); ok && attr.FileAttributes&windows.FILE_ATTRIBUTE_REPARSE_POINT != 0 { + return errf("unknown") + } + return nil +} + +func rejectHardlink(f *os.File, _ os.FileInfo) error { + var info windows.ByHandleFileInformation + if err := windows.GetFileInformationByHandle(windows.Handle(f.Fd()), &info); err != nil { + return err + } + if info.NumberOfLinks != 1 { + return errf("unknown") + } + return nil +} + +func replaceFile(from, to string) error { + source, err := windows.UTF16PtrFromString(from) + if err != nil { + return err + } + target, err := windows.UTF16PtrFromString(to) + if err != nil { + return err + } + return windows.MoveFileEx(source, target, windows.MOVEFILE_REPLACE_EXISTING|windows.MOVEFILE_WRITE_THROUGH) +} + +func syncDirectory(string) error { return nil } diff --git a/internal/dotqueue/json_strict.go b/internal/dotqueue/json_strict.go new file mode 100644 index 0000000..582cbf1 --- /dev/null +++ b/internal/dotqueue/json_strict.go @@ -0,0 +1,72 @@ +package dotqueue + +import ( + "bytes" + "encoding/json" + "io" + "unicode/utf8" +) + +func decodeObject(data []byte, limit int) (map[string]json.RawMessage, error) { + if len(data) > limit || !utf8.Valid(data) { + return nil, errf("corrupt") + } + d := json.NewDecoder(bytes.NewReader(data)) + d.UseNumber() + if err := jsonValue(d, 0); err != nil { + return nil, err + } + if _, err := d.Token(); err != io.EOF { + return nil, errf("corrupt") + } + var obj map[string]json.RawMessage + if json.Unmarshal(data, &obj) != nil || obj == nil { + return nil, errf("corrupt") + } + return obj, nil +} + +func jsonValue(d *json.Decoder, depth int) error { + if depth > 64 { + return errf("corrupt") + } + t, err := d.Token() + if err != nil { + return errf("corrupt") + } + delim, ok := t.(json.Delim) + if !ok { + return nil + } + switch delim { + case '{': + seen := map[string]bool{} + for d.More() { + key, err := d.Token() + if err != nil { + return errf("corrupt") + } + k, ok := key.(string) + if !ok || seen[k] { + return errf("corrupt") + } + seen[k] = true + if err := jsonValue(d, depth+1); err != nil { + return err + } + } + case '[': + for d.More() { + if err := jsonValue(d, depth+1); err != nil { + return err + } + } + default: + return errf("corrupt") + } + end, err := d.Token() + if err != nil || (delim == '{' && end != json.Delim('}')) || (delim == '[' && end != json.Delim(']')) { + return errf("corrupt") + } + return nil +} diff --git a/internal/dotqueue/state.go b/internal/dotqueue/state.go new file mode 100644 index 0000000..1e33a49 --- /dev/null +++ b/internal/dotqueue/state.go @@ -0,0 +1,186 @@ +package dotqueue + +import ( + "encoding/json" + "time" +) + +const ( + stateVersion = 1 + markerName = "dot_queue.initialized" + lockName = "dot_queue.lock" + stateFileName = "queue.state.dq" + maxStateBytes = 64 * 1024 * 1024 + claimTokenBytes = 32 +) + +type jobRecord struct { + ID string `json:"id"` + RequestID string `json:"request_id"` + RequestDigest string `json:"request_digest"` + ClientID string `json:"client_id"` + QueueID string `json:"queue_id"` + Mode string `json:"mode"` + State string `json:"state"` + SessionID string `json:"session_id"` + Task string `json:"task"` + Messages []Message `json:"messages"` + CancelRequested bool `json:"cancel_requested"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + EventID string `json:"event_id"` + WorkerID string `json:"worker_id"` + ClaimToken string `json:"claim_token"` + ResultDigest string `json:"result_digest"` + Result *Result `json:"result,omitempty"` + ErrorCode string `json:"error_code,omitempty"` + Recovered bool `json:"recovered"` +} + +type persistenceState struct { + StateVersion int `json:"state_version"` + Enabled bool `json:"enabled"` + Jobs map[string]*jobRecord `json:"jobs"` + RequestIndex map[string]string `json:"request_index"` + Order []string `json:"order"` +} + +func newPersistenceState(enabled bool) *persistenceState { + return &persistenceState{ + StateVersion: stateVersion, + Enabled: enabled, + Jobs: map[string]*jobRecord{}, + RequestIndex: map[string]string{}, + Order: []string{}, + } +} + +func parsePersistenceState(data []byte) (*persistenceState, error) { + obj, err := decodeObject(data, maxStateBytes) + if err != nil { + return nil, err + } + required := []string{"state_version", "enabled", "jobs", "request_index", "order"} + for _, key := range required { + if _, ok := obj[key]; !ok { + return nil, errf("corrupt") + } + } + for key := range obj { + switch key { + case "state_version", "enabled", "jobs", "request_index", "order": + default: + return nil, errf("corrupt") + } + } + var st persistenceState + if json.Unmarshal(data, &st) != nil { + return nil, errf("corrupt") + } + if st.StateVersion != stateVersion { + return nil, errf("corrupt") + } + if st.Jobs == nil || st.RequestIndex == nil || st.Order == nil { + return nil, errf("corrupt") + } + for id, job := range st.Jobs { + if job == nil || job.ID != id { + return nil, errf("corrupt") + } + if err := validateStoredJob(job); err != nil { + return nil, err + } + } + for key, jobID := range st.RequestIndex { + if st.Jobs[jobID] == nil { + return nil, errf("corrupt") + } + parts := splitRequestKey(key) + if parts[0] != st.Jobs[jobID].ClientID || parts[1] != st.Jobs[jobID].RequestID { + return nil, errf("corrupt") + } + } + if len(st.Order) != len(st.Jobs) { + return nil, errf("corrupt") + } + seen := map[string]bool{} + for _, id := range st.Order { + if st.Jobs[id] == nil || seen[id] { + return nil, errf("corrupt") + } + seen[id] = true + } + return &st, nil +} + +func validateStoredJob(job *jobRecord) error { + if err := validateID(job.ID); err != nil { + return errf("corrupt") + } + if err := validateID(job.RequestID); err != nil { + return errf("corrupt") + } + if err := validateID(job.ClientID); err != nil { + return errf("corrupt") + } + if job.QueueID != job.ClientID { + return errf("corrupt") + } + if err := validateID(job.EventID); err != nil { + return errf("corrupt") + } + return nil +} + +func requestKey(clientID, requestID string) string { + return clientID + "\x00" + requestID +} + +func splitRequestKey(key string) [2]string { + parts := [2]string{"", ""} + for i := 0; i < len(key); i++ { + if key[i] == 0 { + parts[0] = key[:i] + parts[1] = key[i+1:] + break + } + } + return parts +} + +func publicJob(rec *jobRecord) Job { + return Job{ + ID: rec.ID, + RequestID: rec.RequestID, + RequestDigest: rec.RequestDigest, + ClientID: rec.ClientID, + QueueID: rec.QueueID, + Mode: rec.Mode, + State: rec.State, + SessionID: rec.SessionID, + CancelRequested: rec.CancelRequested, + CreatedAt: rec.CreatedAt, + UpdatedAt: rec.UpdatedAt, + Result: rec.Result, + ErrorCode: rec.ErrorCode, + } +} + +func submissionFromRecord(rec *jobRecord) Submission { + return Submission{ + RequestID: rec.RequestID, + Mode: rec.Mode, + SessionID: rec.SessionID, + Messages: append([]Message(nil), rec.Messages...), + Task: rec.Task, + } +} + +func terminalState(state string) bool { + switch state { + case StateCancelled, StateCompleted, StateFailed, StateOutcomeUnknown: + return true + default: + return false + } +} diff --git a/internal/dotqueue/store.go b/internal/dotqueue/store.go new file mode 100644 index 0000000..9ba98ec --- /dev/null +++ b/internal/dotqueue/store.go @@ -0,0 +1,599 @@ +package dotqueue + +import ( + "context" + "crypto/rand" + "encoding/hex" + "encoding/json" + "io" + "os" + "path/filepath" + "strings" + "time" + "unicode" + + "github.com/gofrs/flock" +) + +type Store struct { + dir string + key []byte +} + +func Open(dir string, key []byte) (*Store, error) { + if len(key) != 32 { + return nil, errf("unknown") + } + if !filepath.IsAbs(dir) || len(dir) > 32768 || strings.ContainsFunc(dir, unicode.IsControl) { + return nil, errf("unknown") + } + return &Store{dir: dir, key: append([]byte(nil), key...)}, nil +} + +func (s *Store) Initialize(enabled bool) error { + return s.withLock(true, true, func(st *persistenceState) error { + if st.Jobs == nil { + *st = *newPersistenceState(enabled) + return s.ensureMarker() + } + st.Enabled = enabled + return nil + }) +} + +func (s *Store) SetEnabled(enabled bool) error { + return s.withLock(false, true, func(st *persistenceState) error { + st.Enabled = enabled + return nil + }) +} + +func (s *Store) Submit(clientID string, sub Submission) (Job, error) { + if err := validateID(clientID); err != nil { + return Job{}, err + } + if err := validateSubmission(sub); err != nil { + return Job{}, err + } + digest, err := submissionDigest(sub) + if err != nil { + return Job{}, err + } + var out Job + err = s.withLock(false, true, func(st *persistenceState) error { + if !st.Enabled { + return errf("disabled") + } + key := requestKey(clientID, sub.RequestID) + if existingID, ok := st.RequestIndex[key]; ok { + existing := st.Jobs[existingID] + if existing.RequestDigest != digest { + return errf("conflict") + } + out = publicJob(existing) + return nil + } + if len(st.Jobs) >= maxRetainedJobs { + return errf("conflict") + } + now := time.Now().UTC() + jobID, err := newID("job") + if err != nil { + return err + } + eventID, err := newID("evt") + if err != nil { + return err + } + rec := &jobRecord{ + ID: jobID, + RequestID: sub.RequestID, + RequestDigest: digest, + ClientID: clientID, + QueueID: clientID, + Mode: sub.Mode, + State: StateQueued, + SessionID: sub.SessionID, + Task: sub.Task, + Messages: append([]Message(nil), sub.Messages...), + CreatedAt: now, + UpdatedAt: now, + EventID: eventID, + } + st.Jobs[jobID] = rec + st.RequestIndex[key] = jobID + st.Order = append(st.Order, jobID) + out = publicJob(rec) + return nil + }) + return out, err +} + +func (s *Store) Get(clientID, jobID string) (Job, error) { + if err := validateID(clientID); err != nil { + return Job{}, err + } + if err := validateID(jobID); err != nil { + return Job{}, err + } + var out Job + err := s.withLock(false, false, func(st *persistenceState) error { + rec, err := s.jobForClient(st, clientID, jobID) + if err != nil { + return err + } + out = publicJob(rec) + return nil + }) + return out, err +} + +func (s *Store) ByRequest(clientID, requestID string) (Job, error) { + if err := validateID(clientID); err != nil { + return Job{}, err + } + if err := validateID(requestID); err != nil { + return Job{}, err + } + var out Job + err := s.withLock(false, false, func(st *persistenceState) error { + jobID, ok := st.RequestIndex[requestKey(clientID, requestID)] + if !ok { + return errf("notfound") + } + rec, err := s.jobForClient(st, clientID, jobID) + if err != nil { + return err + } + out = publicJob(rec) + return nil + }) + return out, err +} + +func (s *Store) Claim(queueID, jobID, workerID string) (Claim, error) { + if err := validateID(queueID); err != nil { + return Claim{}, err + } + if err := validateID(jobID); err != nil { + return Claim{}, err + } + if err := validateID(workerID); err != nil { + return Claim{}, err + } + var out Claim + err := s.withLock(false, true, func(st *persistenceState) error { + if !st.Enabled { + return errf("disabled") + } + rec := st.Jobs[jobID] + if rec == nil || rec.QueueID != queueID { + return errf("notfound") + } + if terminalState(rec.State) { + return errf("conflict") + } + if rec.State == StateQueued { + token, err := newClaimToken() + if err != nil { + return err + } + rec.State = StateClaimed + rec.WorkerID = workerID + rec.ClaimToken = token + rec.UpdatedAt = time.Now().UTC() + } else if rec.State == StateClaimed || rec.State == StateRunning || rec.State == StateWaitingUser || rec.State == StateCancelRequested { + if rec.WorkerID != workerID { + return errf("claim_conflict") + } + if rec.ClaimToken == "" { + return errf("corrupt") + } + } else { + return errf("conflict") + } + out = Claim{ + Job: publicJob(rec), + Submission: submissionFromRecord(rec), + Token: rec.ClaimToken, + } + return nil + }) + return out, err +} + +func (s *Store) Progress(jobID, token, state, summary string) (Job, error) { + if err := validateID(jobID); err != nil { + return Job{}, err + } + if err := validateProgressState(state); err != nil { + return Job{}, err + } + if err := validateSummary(summary); err != nil { + return Job{}, err + } + var out Job + err := s.withLock(false, true, func(st *persistenceState) error { + if !st.Enabled { + return errf("disabled") + } + rec := st.Jobs[jobID] + if rec == nil { + return errf("notfound") + } + if rec.ClaimToken != token { + return errf("conflict") + } + switch rec.State { + case StateClaimed, StateRunning, StateWaitingUser, StateCancelRequested: + rec.State = state + if rec.CancelRequested { + rec.State = StateCancelRequested + } + rec.UpdatedAt = time.Now().UTC() + out = publicJob(rec) + return nil + default: + return errf("conflict") + } + }) + return out, err +} + +func (s *Store) Complete(jobID, token string, result Result) (Job, error) { + if err := validateID(jobID); err != nil { + return Job{}, err + } + if err := validateResult(result); err != nil { + return Job{}, err + } + digest, err := resultDigest(result) + if err != nil { + return Job{}, err + } + var out Job + err = s.withLock(false, true, func(st *persistenceState) error { + if !st.Enabled { + return errf("disabled") + } + rec := st.Jobs[jobID] + if rec == nil { + return errf("notfound") + } + if rec.ClaimToken != token { + return errf("conflict") + } + if terminalState(rec.State) { + if rec.ResultDigest == digest { + out = publicJob(rec) + return nil + } + return errf("conflict") + } + if result.Cancelled { + if !rec.CancelRequested { + return errf("conflict") + } + rec.State = StateCancelled + } else if result.ErrorCode != "" { + rec.State = StateFailed + rec.ErrorCode = result.ErrorCode + } else { + rec.State = StateCompleted + } + rec.Result = &Result{ + Text: result.Text, + Artifacts: append([]Artifact(nil), result.Artifacts...), + ErrorCode: result.ErrorCode, + Cancelled: result.Cancelled, + } + rec.ResultDigest = digest + rec.UpdatedAt = time.Now().UTC() + out = publicJob(rec) + return nil + }) + return out, err +} + +func (s *Store) Cancel(clientID, jobID string) (Job, error) { + if err := validateID(clientID); err != nil { + return Job{}, err + } + if err := validateID(jobID); err != nil { + return Job{}, err + } + var out Job + err := s.withLock(false, true, func(st *persistenceState) error { + if !st.Enabled { + return errf("disabled") + } + rec, err := s.jobForClient(st, clientID, jobID) + if err != nil { + return err + } + if terminalState(rec.State) { + out = publicJob(rec) + return nil + } + rec.CancelRequested = true + rec.UpdatedAt = time.Now().UTC() + switch rec.State { + case StateQueued: + rec.State = StateCancelled + default: + rec.State = StateCancelRequested + } + out = publicJob(rec) + return nil + }) + return out, err +} + +func (s *Store) Recover() error { + return s.withLock(false, true, func(st *persistenceState) error { + if !st.Enabled { + return errf("disabled") + } + now := time.Now().UTC() + for _, id := range st.Order { + rec := st.Jobs[id] + if rec == nil || rec.Recovered { + continue + } + switch rec.State { + case StateClaimed, StateRunning, StateWaitingUser, StateCancelRequested: + rec.State = StateOutcomeUnknown + rec.Recovered = true + rec.UpdatedAt = now + } + } + return nil + }) +} + +func (s *Store) Pending(queueID string) ([]Notice, error) { + if err := validateID(queueID); err != nil { + return nil, err + } + var out []Notice + err := s.withLock(false, false, func(st *persistenceState) error { + for _, id := range st.Order { + rec := st.Jobs[id] + if rec == nil || rec.QueueID != queueID || rec.State != StateQueued { + continue + } + out = append(out, Notice{ + EventID: rec.EventID, + QueueID: rec.QueueID, + JobID: rec.ID, + Mode: rec.Mode, + }) + } + return nil + }) + return out, err +} + +func (s *Store) jobForClient(st *persistenceState, clientID, jobID string) (*jobRecord, error) { + rec := st.Jobs[jobID] + if rec == nil || rec.ClientID != clientID { + return nil, errf("notfound") + } + return rec, nil +} + +func (s *Store) withLock(allowCreate bool, mutate bool, fn func(*persistenceState) error) error { + if err := s.checkDir(allowCreate); err != nil { + return err + } + lockPath := filepath.Join(s.dir, lockName) + if err := regularFile(lockPath); err != nil { + return wrapUnknown(err) + } + f, err := os.OpenFile(lockPath, os.O_CREATE|os.O_RDWR, 0600) + if err != nil { + return errf("unknown") + } + info, statErr := f.Stat() + if statErr != nil || rejectHardlink(f, info) != nil { + f.Close() + return errf("unknown") + } + if f.Close() != nil || privatePath(lockPath, false) != nil { + return errf("unknown") + } + lock := flock.New(lockPath) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + ok, err := lock.TryLockContext(ctx, 10*time.Millisecond) + if err != nil || !ok { + return errf("unknown") + } + defer lock.Unlock() + + st, err := s.loadState(allowCreate) + if err != nil { + return err + } + if st == nil { + var empty persistenceState + if err := fn(&empty); err != nil { + return err + } + return s.saveState(&empty) + } + if mutate { + if err := fn(st); err != nil { + return err + } + return s.saveState(st) + } + return fn(st) +} + +func (s *Store) loadState(allowCreate bool) (*persistenceState, error) { + marker := filepath.Join(s.dir, markerName) + _, markerErr := os.Lstat(marker) + statePath := filepath.Join(s.dir, stateFileName) + _, stateErr := os.Lstat(statePath) + if os.IsNotExist(markerErr) && os.IsNotExist(stateErr) { + if allowCreate { + return nil, nil + } + return nil, errf("corrupt") + } + if markerErr != nil || stateErr != nil { + if os.IsNotExist(stateErr) && markerErr == nil { + return nil, errf("corrupt") + } + return nil, errf("corrupt") + } + if err := regularFile(statePath); err != nil { + return nil, wrapUnknown(err) + } + f, err := os.Open(statePath) + if err != nil { + return nil, errf("corrupt") + } + defer f.Close() + info, err := f.Stat() + if err != nil || !info.Mode().IsRegular() || rejectHardlink(f, info) != nil { + return nil, errf("corrupt") + } + if info.Size() > maxStateBytes { + return nil, errf("corrupt") + } + data, err := io.ReadAll(io.LimitReader(f, maxStateBytes+1)) + if err != nil { + return nil, errf("corrupt") + } + plain, err := openEnvelope(s.key, data) + if err != nil { + return nil, err + } + return parsePersistenceState(plain) +} + +func (s *Store) saveState(st *persistenceState) error { + if st.StateVersion != stateVersion { + return errf("corrupt") + } + data, err := json.Marshal(st) + if err != nil || len(data) > maxStateBytes { + return errf("unknown") + } + sealed, err := seal(s.key, data) + if err != nil { + return err + } + statePath := filepath.Join(s.dir, stateFileName) + if err := regularFile(statePath); err != nil { + return wrapUnknown(err) + } + f, err := os.CreateTemp(s.dir, ".dot-queue-*") + if err != nil { + return errf("unknown") + } + tmp := f.Name() + defer os.Remove(tmp) + if privatePath(tmp, false) != nil { + f.Close() + return errf("unknown") + } + n, writeErr := f.Write(sealed) + syncErr := f.Sync() + closeErr := f.Close() + if writeErr != nil || n != len(sealed) || syncErr != nil || closeErr != nil { + return errf("unknown") + } + if regularFile(statePath) != nil || replaceFile(tmp, statePath) != nil || syncDirectory(s.dir) != nil { + return errf("unknown") + } + return nil +} + +func (s *Store) ensureMarker() error { + path := filepath.Join(s.dir, markerName) + if err := regularFile(path); err != nil { + return wrapUnknown(err) + } + f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600) + if os.IsExist(err) { + f, err = os.Open(path) + if err != nil { + return errf("unknown") + } + info, statErr := f.Stat() + if statErr != nil || rejectHardlink(f, info) != nil { + f.Close() + return errf("unknown") + } + if f.Close() != nil || privatePath(path, false) != nil { + return errf("unknown") + } + return nil + } + if err != nil { + return errf("unknown") + } + if privatePath(path, false) != nil { + f.Close() + return errf("unknown") + } + _, writeErr := f.WriteString("1\n") + syncErr := f.Sync() + closeErr := f.Close() + if writeErr != nil || syncErr != nil || closeErr != nil || syncDirectory(s.dir) != nil { + return errf("unknown") + } + return nil +} + +func (s *Store) checkDir(create bool) error { + if !filepath.IsAbs(s.dir) || len(s.dir) > 32768 || strings.ContainsFunc(s.dir, unicode.IsControl) { + return errf("unknown") + } + if create { + if os.MkdirAll(s.dir, 0700) != nil { + return errf("unknown") + } + } + info, err := os.Lstat(s.dir) + if os.IsNotExist(err) { + return errf("corrupt") + } + if err != nil || !info.IsDir() || rejectLink(info) != nil { + return errf("unknown") + } + if create && privatePath(s.dir, true) != nil { + return errf("unknown") + } + return nil +} + +func regularFile(path string) error { + info, err := os.Lstat(path) + if os.IsNotExist(err) { + return nil + } + if err != nil || !info.Mode().IsRegular() || rejectLink(info) != nil { + return errf("unknown") + } + return nil +} + +func newID(prefix string) (string, error) { + var b [16]byte + if _, err := rand.Read(b[:]); err != nil { + return "", errf("unknown") + } + return prefix + "." + hex.EncodeToString(b[:]), nil +} + +func newClaimToken() (string, error) { + var b [claimTokenBytes]byte + if _, err := rand.Read(b[:]); err != nil { + return "", errf("unknown") + } + return hex.EncodeToString(b[:]), nil +} diff --git a/internal/dotqueue/store_test.go b/internal/dotqueue/store_test.go new file mode 100644 index 0000000..ceebdd9 --- /dev/null +++ b/internal/dotqueue/store_test.go @@ -0,0 +1,307 @@ +package dotqueue + +import ( + "bytes" + "fmt" + "os" + "path/filepath" + "sync" + "testing" +) + +var testKey = []byte{ + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, + 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, + 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, + 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, 0x20, +} + +func openTestStore(t *testing.T) (*Store, string) { + dir := t.TempDir() + abs, err := filepath.Abs(dir) + if err != nil { + t.Fatal(err) + } + store, err := Open(abs, testKey) + if err != nil { + t.Fatal(err) + } + if err := store.Initialize(true); err != nil { + t.Fatal(err) + } + return store, abs +} + +func sampleSubmission(requestID string) Submission { + return Submission{ + RequestID: requestID, + Mode: ModeAnswer, + SessionID: "sess.1", + Messages: []Message{{Role: "user", Content: "hello dot"}}, + } +} + +func TestEncryptedRoundTripAndReopen(t *testing.T) { + store, dir := openTestStore(t) + job, err := store.Submit("client.a", sampleSubmission("req.1")) + if err != nil { + t.Fatal(err) + } + if job.State != StateQueued || job.QueueID != "client.a" { + t.Fatalf("unexpected job %#v", job) + } + reopened, err := Open(dir, testKey) + if err != nil { + t.Fatal(err) + } + again, err := reopened.ByRequest("client.a", "req.1") + if err != nil || again.ID != job.ID { + t.Fatalf("persisted lookup failed: %v %#v", err, again) + } + data, err := os.ReadFile(filepath.Join(dir, stateFileName)) + if err != nil { + t.Fatal(err) + } + if bytes.Contains(data, []byte("hello dot")) { + t.Fatal("plaintext prompt found on disk") + } +} + +func TestSubmitIdempotencyAndConflict(t *testing.T) { + store, _ := openTestStore(t) + sub := sampleSubmission("req.idem") + first, err := store.Submit("client.a", sub) + if err != nil { + t.Fatal(err) + } + second, err := store.Submit("client.a", sub) + if err != nil || second.ID != first.ID { + t.Fatalf("idempotent submit failed: %v %#v", err, second) + } + changed := sub + changed.Messages[0].Content = "changed" + _, err = store.Submit("client.a", changed) + if errorCode(err) != "conflict" { + t.Fatalf("expected conflict, got %v", err) + } +} + +func TestClaimLifecycleAndResultIdempotency(t *testing.T) { + store, _ := openTestStore(t) + job, err := store.Submit("client.a", sampleSubmission("req.claim")) + if err != nil { + t.Fatal(err) + } + notices, err := store.Pending("client.a") + if err != nil || len(notices) != 1 || notices[0].JobID != job.ID { + t.Fatalf("pending notice mismatch: %v %#v", err, notices) + } + claim1, err := store.Claim("client.a", job.ID, "worker.1") + if err != nil { + t.Fatal(err) + } + claim2, err := store.Claim("client.a", job.ID, "worker.1") + if err != nil || claim2.Token != claim1.Token { + t.Fatalf("duplicate worker claim failed: %v", err) + } + _, err = store.Claim("client.a", job.ID, "worker.2") + if errorCode(err) != "claim_conflict" { + t.Fatalf("expected claim_conflict, got %v", err) + } + _, err = store.Progress(job.ID, claim1.Token, StateRunning, "working") + if err != nil { + t.Fatal(err) + } + res := Result{Text: "done", Artifacts: []Artifact{{URL: "https://example.com/a", Label: "a"}}} + done1, err := store.Complete(job.ID, claim1.Token, res) + if err != nil || done1.State != StateCompleted { + t.Fatalf("complete failed: %v %#v", err, done1) + } + done2, err := store.Complete(job.ID, claim1.Token, res) + if err != nil || done2.State != StateCompleted { + t.Fatalf("idempotent complete failed: %v", err) + } + res.Text = "different" + _, err = store.Complete(job.ID, claim1.Token, res) + if errorCode(err) != "conflict" { + t.Fatalf("expected result conflict, got %v", err) + } +} + +func TestCancelIntentAndRecover(t *testing.T) { + store, _ := openTestStore(t) + queued, err := store.Submit("client.a", sampleSubmission("req.cancel.queued")) + if err != nil { + t.Fatal(err) + } + cancelled, err := store.Cancel("client.a", queued.ID) + if err != nil || cancelled.State != StateCancelled { + t.Fatalf("queued cancel failed: %v %#v", err, cancelled) + } + active, err := store.Submit("client.a", sampleSubmission("req.cancel.active")) + if err != nil { + t.Fatal(err) + } + claim, err := store.Claim("client.a", active.ID, "worker.1") + if err != nil { + t.Fatal(err) + } + requested, err := store.Cancel("client.a", active.ID) + if err != nil || requested.State != StateCancelRequested || !requested.CancelRequested { + t.Fatalf("active cancel failed: %v %#v", err, requested) + } + _, err = store.Progress(active.ID, claim.Token, StateRunning, "still running") + if err != nil { + t.Fatal(err) + } + got, err := store.Get("client.a", active.ID) + if err != nil || got.State != StateCancelRequested { + t.Fatalf("progress preserved cancel intent: %v %#v", err, got) + } + if err := store.Recover(); err != nil { + t.Fatal(err) + } + recovered, err := store.Get("client.a", active.ID) + if err != nil || recovered.State != StateOutcomeUnknown { + t.Fatalf("recover failed: %v %#v", err, recovered) + } + if err := store.Recover(); err != nil { + t.Fatal(err) + } + again, err := store.Get("client.a", active.ID) + if err != nil || again.State != StateOutcomeUnknown { + t.Fatalf("recover should be once: %#v", again) + } +} + +func TestDisabledRejectsSensitiveOps(t *testing.T) { + store, _ := openTestStore(t) + if err := store.SetEnabled(false); err != nil { + t.Fatal(err) + } + _, err := store.Submit("client.a", sampleSubmission("req.disabled")) + if errorCode(err) != "disabled" { + t.Fatalf("expected disabled, got %v", err) + } + if err := store.SetEnabled(true); err != nil { + t.Fatal(err) + } + job, err := store.Submit("client.a", sampleSubmission("req.enabled")) + if err != nil { + t.Fatal(err) + } + if err := store.SetEnabled(false); err != nil { + t.Fatal(err) + } + _, err = store.Claim("client.a", job.ID, "worker.1") + if errorCode(err) != "disabled" { + t.Fatalf("claim should be disabled: %v", err) + } + _, err = store.Get("client.a", job.ID) + if err != nil { + t.Fatalf("read should work while disabled: %v", err) + } +} + +func TestMissingStateFailsClosed(t *testing.T) { + store, dir := openTestStore(t) + if err := os.WriteFile(filepath.Join(dir, markerName), []byte("1\n"), 0600); err != nil { + t.Fatal(err) + } + if err := os.Remove(filepath.Join(dir, stateFileName)); err != nil { + t.Fatal(err) + } + _, err := store.Get("client.a", "job.missing") + if errorCode(err) != "corrupt" { + t.Fatalf("expected corrupt for missing state, got %v", err) + } +} + +func TestCorruptEncryptedState(t *testing.T) { + store, dir := openTestStore(t) + _, err := store.Submit("client.a", sampleSubmission("req.corrupt")) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, stateFileName), []byte("DQ1bad"), 0600); err != nil { + t.Fatal(err) + } + _, err = store.ByRequest("client.a", "req.corrupt") + if errorCode(err) != "corrupt" { + t.Fatalf("expected corrupt, got %v", err) + } +} + +func TestBoundsValidation(t *testing.T) { + store, _ := openTestStore(t) + big := sampleSubmission("req.big") + big.Messages = make([]Message, maxMessages+1) + _, err := store.Submit("client.a", big) + if errorCode(err) != "unknown" { + t.Fatalf("expected bound rejection, got %v", err) + } + taskOnly := Submission{RequestID: "req.task", Mode: ModeTask, Task: "do thing"} + _, err = store.Submit("client.a", taskOnly) + if err != nil { + t.Fatalf("task mode should be valid: %v", err) + } + emptyTask := Submission{RequestID: "req.empty", Mode: ModeTask, Task: " "} + _, err = store.Submit("client.a", emptyTask) + if errorCode(err) != "unknown" { + t.Fatalf("empty task rejected: %v", err) + } +} + +func TestConcurrentSubmitAndClaim(t *testing.T) { + store, dir := openTestStore(t) + other, err := Open(dir, testKey) + if err != nil { + t.Fatal(err) + } + var wg sync.WaitGroup + const n = 16 + ids := make([]string, n) + for i := 0; i < n; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + reqID := fmt.Sprintf("req.parallel.%d", i) + job, err := store.Submit("client.a", sampleSubmission(reqID)) + if err != nil { + t.Errorf("submit %d: %v", i, err) + return + } + ids[i] = job.ID + }(i) + } + wg.Wait() + wg.Add(1) + go func() { + defer wg.Done() + for i := 0; i < n; i++ { + if ids[i] == "" { + continue + } + _, err := other.Claim("client.a", ids[i], "worker.1") + if err != nil { + t.Errorf("claim %d: %v", i, err) + } + } + }() + wg.Wait() +} + +func TestMaxRetainedJobs(t *testing.T) { + store, _ := openTestStore(t) + for i := 0; i < maxRetainedJobs; i++ { + req := sampleSubmission(fmt.Sprintf("req.cap.%d", i)) + if _, err := store.Submit("client.a", req); err != nil { + t.Fatalf("fill %d: %v", i, err) + } + } + _, err := store.Submit("client.a", sampleSubmission("req.overflow")) + if errorCode(err) != "conflict" { + t.Fatalf("expected capacity conflict, got %v", err) + } +} + diff --git a/internal/dotqueue/types.go b/internal/dotqueue/types.go new file mode 100644 index 0000000..58e162b --- /dev/null +++ b/internal/dotqueue/types.go @@ -0,0 +1,72 @@ +package dotqueue + +import "time" + +const ( + ModeAnswer = "answer" + ModeTask = "task" + + StateQueued = "queued" + StateClaimed = "claimed" + StateRunning = "running" + StateWaitingUser = "waiting_user" + StateCancelRequested = "cancel_requested" + StateCancelled = "cancelled" + StateCompleted = "completed" + StateFailed = "failed" + StateOutcomeUnknown = "outcome_unknown" +) + +type Submission struct { + RequestID string `json:"request_id"` + Mode string `json:"mode"` + SessionID string `json:"session_id"` + Messages []Message `json:"messages"` + Task string `json:"task"` +} + +type Message struct { + Role string `json:"role"` + Content string `json:"content"` +} + +type Job struct { + ID string `json:"id"` + RequestID string `json:"request_id"` + RequestDigest string `json:"request_digest"` + ClientID string `json:"client_id"` + QueueID string `json:"queue_id"` + Mode string `json:"mode"` + State string `json:"state"` + SessionID string `json:"session_id"` + CancelRequested bool `json:"cancel_requested"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + Result *Result `json:"result,omitempty"` + ErrorCode string `json:"error_code,omitempty"` +} + +type Claim struct { + Job Job `json:"job"` + Submission Submission `json:"submission"` + Token string `json:"token"` +} + +type Result struct { + Text string `json:"text"` + Artifacts []Artifact `json:"artifacts"` + ErrorCode string `json:"error_code"` + Cancelled bool `json:"cancelled"` +} + +type Artifact struct { + URL string `json:"url"` + Label string `json:"label"` +} + +type Notice struct { + EventID string `json:"event_id"` + QueueID string `json:"queue_id"` + JobID string `json:"job_id"` + Mode string `json:"mode"` +} diff --git a/internal/dotqueue/validate.go b/internal/dotqueue/validate.go new file mode 100644 index 0000000..b67ff3f --- /dev/null +++ b/internal/dotqueue/validate.go @@ -0,0 +1,119 @@ +package dotqueue + +import ( + "net/url" + "regexp" + "strings" + "unicode/utf8" +) + +const ( + maxMessages = 32 + maxPromptBytes = 256 * 1024 + maxResultBytes = 256 * 1024 + maxSummaryBytes = 2048 + maxArtifacts = 16 + maxLabelBytes = 256 + maxRetainedJobs = 100 + maxIDLen = 128 +) + +var idPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$`) + +var allowedRoles = map[string]bool{ + "user": true, + "assistant": true, + "system": true, +} + +func validateID(id string) error { + if !idPattern.MatchString(id) { + return errf("unknown") + } + return nil +} + +func validateSubmission(sub Submission) error { + if err := validateID(sub.RequestID); err != nil { + return err + } + if sub.Mode != ModeAnswer && sub.Mode != ModeTask { + return errf("unknown") + } + if sub.Mode == ModeAnswer && len(sub.Messages) == 0 { + return errf("unknown") + } + if sub.Mode == ModeTask && strings.TrimSpace(sub.Task) == "" { + return errf("unknown") + } + if len(sub.Messages) > maxMessages { + return errf("unknown") + } + var promptBytes int + for _, m := range sub.Messages { + if !allowedRoles[m.Role] { + return errf("unknown") + } + promptBytes += len(m.Content) + } + if promptBytes > maxPromptBytes { + return errf("unknown") + } + if sub.SessionID != "" { + if err := validateID(sub.SessionID); err != nil { + return err + } + } + return nil +} + +func validateProgressState(state string) error { + switch state { + case StateRunning, StateWaitingUser: + return nil + default: + return errf("unknown") + } +} + +func validateSummary(summary string) error { + if len(summary) > maxSummaryBytes { + return errf("unknown") + } + if !utf8.ValidString(summary) { + return errf("unknown") + } + return nil +} + +func validateResult(res Result) error { + if len(res.Text) > maxResultBytes { + return errf("unknown") + } + if !utf8.ValidString(res.Text) { + return errf("unknown") + } + if len(res.Artifacts) > maxArtifacts { + return errf("unknown") + } + for _, a := range res.Artifacts { + if err := validateArtifact(a); err != nil { + return err + } + } + return nil +} + +func validateArtifact(a Artifact) error { + if len(a.Label) > maxLabelBytes || !utf8.ValidString(a.Label) { + return errf("unknown") + } + u, err := url.Parse(a.URL) + if err != nil || u.Scheme != "https" || u.Host == "" { + return errf("unknown") + } + if strings.Contains(a.URL, "\\") || strings.HasPrefix(a.URL, "file:") { + return errf("unknown") + } + return nil +} From 1144745367f3352474198c239d80015e3a37a34b Mon Sep 17 00:00:00 2001 From: Pyuyi <204892921+VeraPyuyi@users.noreply.github.com> Date: Mon, 19 Oct 2026 11:23:24 +0800 Subject: [PATCH 04/47] fix(dotqueue): enforce reviewed queue security and lifecycle invariants Address round1 I1-I7 and M1: validate worker capabilities, reconcile unknown outcomes, gate sensitive reads, bound all UTF-8 content, strictly validate authenticated state, enforce marker/privacy checks and lock the actual validated handle. Persist progress summaries and authenticate envelope headers. Behavioral RED retained: dotqueue-round1-red (exit 1 before fixes), cancel-red and url-red (exit 1). Final GREEN: isolated runner --race go test ./internal/dotqueue -race -count=1 -v, exit 0; dotqueue-round1-final-race.log/.json. Includes actual child-process submit/claim CAS and Windows ACL/junction fixtures. Initial race filesystem failure diagnosed as MoveFileEx access denied; bounded replacement retry added. No full-suite rerun; independent review pending. --- docs/DOT_TASK_QUEUE.md | 43 +-- internal/dotqueue/contract_test.go | 293 +++++++++++++++++++ internal/dotqueue/crypto.go | 4 +- internal/dotqueue/files_unix.go | 9 + internal/dotqueue/files_unix_test.go | 45 +++ internal/dotqueue/files_windows.go | 25 +- internal/dotqueue/files_windows_test.go | 87 ++++++ internal/dotqueue/paths.go | 75 +++++ internal/dotqueue/review_test.go | 364 ++++++++++++++++++++++++ internal/dotqueue/schema.go | 79 +++++ internal/dotqueue/state.go | 104 ++++++- internal/dotqueue/store.go | 78 +++-- internal/dotqueue/store_test.go | 7 +- internal/dotqueue/types.go | 1 + internal/dotqueue/validate.go | 142 ++++----- 15 files changed, 1221 insertions(+), 135 deletions(-) create mode 100644 internal/dotqueue/contract_test.go create mode 100644 internal/dotqueue/files_unix_test.go create mode 100644 internal/dotqueue/files_windows_test.go create mode 100644 internal/dotqueue/paths.go create mode 100644 internal/dotqueue/review_test.go create mode 100644 internal/dotqueue/schema.go diff --git a/docs/DOT_TASK_QUEUE.md b/docs/DOT_TASK_QUEUE.md index 600f8ff..bf2ad9f 100644 --- a/docs/DOT_TASK_QUEUE.md +++ b/docs/DOT_TASK_QUEUE.md @@ -1,42 +1,27 @@ -# Dot task queue (`internal/dotqueue`) +# Dot task queue -Task 1 delivers a durable, encrypted filesystem queue for DotControl routing work. Network exposure, MCP tools, vault wiring, and runtime gates are intentionally out of scope here. +`internal/dotqueue` is a trusted local Go backend. It persists work and stable event notices; it does not expose routing, implement authentication, fetch artifact URLs, or establish Dot readiness. -## API surface +Call `Open(dir, key)` with a caller-managed 32-byte key and explicitly call `Initialize(enabled)` for a new store. Open copies the key and never persists it or automatically recovers work. Initialize on an existing valid store sets enabled; a missing, malformed, or unsupported initialized marker/state fails closed. `SetEnabled` changes the persisted gate. `ConfigStatus()` returns only `Enabled` and remains available while disabled. Get, ByRequest, Pending, Submit, Claim, Progress, Complete, Cancel and Recover all check enabled under the transaction lock. -- `Open(dir, key)` — absolute state directory and 32-byte AES-256-GCM key supplied by caller (never persisted). -- `Store.Initialize(enabled)` — explicit first-time setup; writes `dot_queue.initialized` and refuses silent re-init if state is missing after the marker exists. -- `Store.SetEnabled(bool)` — toggles whether sensitive mutations are accepted. -- `Submit`, `Get`, `ByRequest`, `Claim`, `Progress`, `Complete`, `Cancel`, `Recover`, `Pending` — semantics match `docs/superpowers/specs/2026-10-19-dot-routing-design.md` and the task brief. +Submission IDs (including client, queue, worker and optional session IDs) use `^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$`. Answer mode requires messages; task mode requires nonblank Task. The supported message roles are system, user and assistant. At most 32 messages are accepted. Combined Task and message content is at most 262144 UTF-8 bytes. Invalid UTF-8 is rejected before hashing. Canonical JSON uses fixed field order and empty arrays for nil message/artifact lists; SHA-256 identifies the complete submission/result. -Exported structs use JSON snake_case field names. Public `Job` values omit claim tokens and other worker secrets. +Same client/request ID with identical content returns the original job; changed content conflicts. Queue ID is derived from client ID. Retained capacity is exactly 100 jobs, including terminal jobs, with no eviction or request-ID reuse. Acceptance and a single stable EventID are written atomically. Pending returns queued notices; retries and reopening preserve their IDs. Delivery/notification acknowledgment belongs to a later slice. -## Storage and concurrency +Claim binds a random 256-bit token to a job and worker. Same-worker retries return the original token; competing workers conflict. Worker mutations reject empty, malformed and incorrect tokens, and Complete requires a claimed lifecycle source. Public Job omits token and original submission; Claim contains both and must not be logged or forwarded as a public job response. -- Private directory (`0700`) and files (`0600`) with Windows current-user DACL, matching `internal/historyaccess` patterns. -- Cross-process `flock` on `dot_queue.lock` (5s acquisition budget, 10ms retry interval). -- All queue state is held in a single encrypted blob `queue.state.dq` (`D` + version byte + nonce + AES-GCM ciphertext). Prompts, results, and claim tokens are not stored in plaintext on disk. -- Updates are read → validate → mutate → temp file → sync → atomic replace under the lock. +Progress accepts claimed, running or waiting_user, stores `Job.Summary` durably (maximum 2048 UTF-8 bytes), and preserves cancel_requested while intent remains set. Cancel before claim yields cancelled; after claim it records intent. Cancellation of outcome_unknown retains uncertainty while recording intent. A worker may acknowledge cancellation only with `Result.Cancelled=true` after intent is recorded. A result without that acknowledgment can still complete/ fail while retaining the intent flag. -## Lifecycle rules +Recover is an explicit owner-startup action, never a routine client Open action. It changes claimed/running/waiting_user/cancel_requested to outcome_unknown once, preserving token and notice identity. Unknown work cannot be claimed or progressed into an active state. The original token can reconcile via Complete, including explicit cancellation acknowledgment. Final completed/failed/cancelled results permit identical retries and reject changed results. No lease, automatic reassignment or automatic execution retry exists. -- **Idempotency:** same `client_id` + `request_id` + submission digest returns the same job; digest mismatch is `dot_queue_conflict`. -- **Capacity:** at most 100 retained jobs; overflow returns conflict (no eviction). -- **Claim:** `queued` → `claimed` with a 256-bit token; duplicate claim by the same worker returns the same token; another worker gets `dot_queue_claim_conflict`. -- **Cancel:** before claim → `cancelled`; after claim → `cancel_requested` (progress does not clear cancel intent). -- **Complete:** terminal and idempotent on identical result digest; mismatched terminal payloads conflict. -- **Recover:** explicit only (not on `Open`); moves in-flight worker states to `outcome_unknown` once per job after restart uncertainty. -- **Disabled:** sensitive operations return `dot_queue_disabled`; read APIs remain available. +Result content has a total 262144-byte budget across Text, ErrorCode, artifact URLs and labels. ErrorCode, when present, uses the ID syntax (maximum 128 bytes), representing a stable caller-defined error identifier rather than diagnostic prose. Cancelled and ErrorCode cannot both be set. At most 16 artifacts are allowed, each label at most 256 bytes and URL at most 2048 bytes. All strings must be valid UTF-8. Artifacts require HTTPS without userinfo; localhost/local names, single-label names, private/loopback/link-local/unspecified/non-unicast literal addresses are rejected. Hostnames are not DNS-resolved here. Any future fetcher must validate DNS addresses at connection time and constrain redirects; this backend's syntactic policy is not a network safety guarantee. -## Error codes +State is AES-256-GCM encrypted with a random nonce. The `D`/version header is authenticated as additional data; encrypted JSON also requires state_version 1. Nested schemas reject unknown, duplicate, missing required, null and wrongly typed fields. Every loaded job validates payload/result budgets and digests, lifecycle/claim/cancellation/result consistency, timestamps, unique events/tokens, capacity, order and the job/request-index bijection. Empty arrays are persisted explicitly. Earlier development fixtures without Summary or header AAD are incompatible and fail closed; there is no silent migration or reset. -Stable `dotqueue.Error.Code` values include: `disabled`, `conflict`, `notfound`, `corrupt`, `claim_conflict`, `unknown`. +Every transaction validates a canonical local absolute non-root directory and all existing ancestors against symlinks/reparse points. UNC/device paths, alternate streams, relative paths and noncanonical paths are rejected. Parents must be trusted directories administered by the user or OS; callers must select a dedicated queue directory. Directory privacy is reapplied for every operation (0700 on Unix, protected current-user-only DACL on Windows), as is 0600/current-user privacy on owned files. Marker contents must be exactly `1\n` and every load checks regularity, link count and version. State, marker and lock reject hardlinks and symlinks/reparse points. -## Tests +The installed flock version exposes only pathname-based constructors and keeps its handle private. The queue therefore locks its own validated open handle using OS flock on Unix or LockFileEx on Windows, with a five-second acquisition budget, and checks that the locked handle still identifies the current lock path. This prevents the previous close-and-reopen gap. Read/validate/mutate/encrypt/temp write/sync/atomic replacement is one locked transaction. Windows uses MoveFileEx with WRITE_THROUGH; Unix uses rename plus directory sync. All temporary payloads are ciphertext. Private parents and ACLs protect against other OS principals; this design does **not** isolate a malicious same-user process that can replace directories, inspect process memory or obtain the caller's key. Path checks do not claim adversarial same-user race resistance. -Package tests cover encryption round-trip, reopen persistence, idempotency, claim/complete conflicts, cancel/recover, disabled gating, corrupt/missing state, bounds, concurrency across two `Store` handles, and capacity limits. Run: +Errors are sanitized `*dotqueue.Error`, with short `Code` values such as disabled, conflict, notfound, corrupt, claim_conflict and unknown; Error() adds `dot_queue_`. A nonnil error always means callers must discard any returned value. This API does not authenticate clientID/workerID arguments: future adapters must derive these identities from authenticated scope and keep Initialize/SetEnabled/Recover privileged. -```text -go test ./internal/dotqueue/ -count=1 -python run_checks.py dotqueue-race go test ./internal/dotqueue/ -race -count=1 -``` +Tests use temporary directories and a fixed fake key. They include compile-safe behavioral regression cases, authenticated malformed-state fixtures, Windows ACL repair/reparse fixtures, actual child-process same-request submission/competing claim, literal size and capacity boundaries, persisted results and cancellation acknowledgment, encryption/tamper checks, and two-handle races. Run the package with the isolated runner and `go test ./internal/dotqueue -count=1 -v`; enable CGO and a compiler for `-race`. Exact retained RED/GREEN commands and review mappings are in the task report. No fixture establishes live account or routing readiness. diff --git a/internal/dotqueue/contract_test.go b/internal/dotqueue/contract_test.go new file mode 100644 index 0000000..80db6c2 --- /dev/null +++ b/internal/dotqueue/contract_test.go @@ -0,0 +1,293 @@ +package dotqueue + +import ( + "bytes" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "sync" + "testing" +) + +func TestContractAllIDFieldsAndTenants(t *testing.T) { + s, _ := openTestStore(t) + j := mustSubmit(t, s, "ids") + for _, id := range []string{"", "../escape", strings.Repeat("x", 129), "x/y", "x\x00y"} { + for name, op := range map[string]func() error{ + "client": func() error { _, e := s.Submit(id, sampleSubmission("good")); return e }, "request": func() error { _, e := s.Submit("client.a", sampleSubmission(id)); return e }, "session": func() error { + sub := sampleSubmission("session") + sub.SessionID = id + if id == "" { + sub.SessionID = "bad/path" + } + _, e := s.Submit("client.a", sub) + return e + }, "queue": func() error { _, e := s.Claim(id, j.ID, "worker"); return e }, "worker": func() error { _, e := s.Claim("client.a", j.ID, id); return e }, "job": func() error { _, e := s.Get("client.a", id); return e }} { + if e := op(); e == nil { + t.Errorf("%s accepted %q", name, id) + } + } + } + if _, e := s.Get("client.b", j.ID); e == nil { + t.Error("tenant Get") + } + if _, e := s.ByRequest("client.b", j.RequestID); e == nil { + t.Error("tenant ByRequest") + } + if _, e := s.Cancel("client.b", j.ID); e == nil { + t.Error("tenant Cancel") + } + if _, e := s.Claim("client.b", j.ID, "worker"); e == nil { + t.Error("tenant Claim") + } + if ns, e := s.Pending("client.b"); e != nil || len(ns) != 0 { + t.Error("tenant Pending") + } +} +func TestContractCapacityLiteralAndRetainedRequests(t *testing.T) { + s, dir := openTestStore(t) + first := mustSubmit(t, s, "capacity.0") + for i := 1; i < 100; i++ { + mustSubmit(t, s, fmt.Sprintf("capacity.%d", i)) + } + if _, e := s.Cancel("client.a", first.ID); e != nil { + t.Fatal(e) + } + s = reopen(t, dir) + if _, e := s.Submit("client.a", sampleSubmission("overflow")); e == nil { + t.Error("101st job accepted") + } + j, e := s.Submit("client.a", sampleSubmission("capacity.0")) + if e != nil || j.ID != first.ID { + t.Errorf("retained request: %v %#v", e, j) + } + sub := sampleSubmission("capacity.0") + sub.Messages[0].Content = "different" + if _, e = s.Submit("client.a", sub); e == nil { + t.Error("retained request reused") + } +} +func TestContractTwoStoreSameRequestRace(t *testing.T) { + s, dir := openTestStore(t) + other := reopen(t, dir) + var wg sync.WaitGroup + var jobs [2]Job + var errs [2]error + start := make(chan struct{}) + for i, store := range []*Store{s, other} { + wg.Add(1) + go func(i int, store *Store) { + defer wg.Done() + <-start + jobs[i], errs[i] = store.Submit("client.a", sampleSubmission("same")) + }(i, store) + } + close(start) + wg.Wait() + if errs[0] != nil || errs[1] != nil || jobs[0].ID != jobs[1].ID { + t.Fatalf("CAS jobs %#v errors %v", jobs, errs) + } +} +func TestContractAuthenticatedNestedAndLifecycle(t *testing.T) { + for _, kind := range []string{"result-extra", "artifact-extra", "result-null", "text-null", "token", "worker", "result-digest", "result-mismatch", "result-cancel", "index-extra", "duplicate-order", "null-job", "over-capacity"} { + t.Run(kind, func(t *testing.T) { + s, _ := openTestStore(t) + j := mustSubmit(t, s, "nested") + c, e := s.Claim("client.a", j.ID, "worker") + if e != nil { + t.Fatal(e) + } + if _, e = s.Complete(j.ID, c.Token, Result{Text: "done", Artifacts: []Artifact{{URL: "https://example.com/a", Label: "a"}}}); e != nil { + t.Fatal(e) + } + authenticatedFixture(t, s, func(m map[string]any) { + jobs := m["jobs"].(map[string]any) + r := jobs[j.ID].(map[string]any) + switch kind { + case "result-extra": + r["result"].(map[string]any)["extra"] = true + case "artifact-extra": + r["result"].(map[string]any)["artifacts"].([]any)[0].(map[string]any)["extra"] = true + case "result-null": + r["result"] = nil + case "text-null": + r["result"].(map[string]any)["text"] = nil + case "token": + r["claim_token"] = "" + case "worker": + r["worker_id"] = "bad/id" + case "result-digest": + r["result_digest"] = strings.Repeat("a", 64) + case "result-mismatch": + r["state"] = "failed" + case "result-cancel": + r["cancel_requested"] = false + r["state"] = "cancelled" + case "index-extra": + m["request_index"].(map[string]any)["extra"] = j.ID + case "duplicate-order": + m["order"] = []any{j.ID, j.ID} + case "null-job": + jobs[j.ID] = nil + case "over-capacity": + for i := 0; i < 101; i++ { + jobs[fmt.Sprint(i)] = r + } + } + }) + if _, e = s.Get("client.a", j.ID); e == nil || e.Error() != "dot_queue_corrupt" { + t.Errorf("invalid state %v", e) + } + }) + } + s, _ := openTestStore(t) + j := mustSubmit(t, s, "raw-json") + for _, suffix := range []string{" {}", "\nnull"} { + b, _ := os.ReadFile(filepath.Join(s.dir, stateFileName)) + plain, e := openEnvelope(testKey, b) + if e != nil { + t.Fatal(e) + } + bad, _ := seal(testKey, append(plain, []byte(suffix)...)) + os.WriteFile(filepath.Join(s.dir, stateFileName), bad, 0600) + if _, e = s.Get("client.a", j.ID); e == nil { + t.Error("trailing JSON accepted") + } + os.WriteFile(filepath.Join(s.dir, stateFileName), b, 0600) + } +} +func TestContractArtifactLocalAliases(t *testing.T) { + for _, u := range []string{"https://127.1/a", "https://2130706433/a", "https://100.64.0.1/a", "https://example.com:0/a", "https://example.com:99999/a"} { + if validateArtifact(Artifact{URL: u}) == nil { + t.Errorf("local/invalid artifact accepted %s", u) + } + } +} +func TestContractBoundsMetadataAndEnvelope(t *testing.T) { + for _, n := range []int{32, 33} { + sub := sampleSubmission("messages") + sub.Messages = make([]Message, n) + for i := range sub.Messages { + sub.Messages[i] = Message{Role: "user", Content: "a"} + } + if e := validateSubmission(sub); (e == nil) != (n == 32) { + t.Errorf("message count %d %v", n, e) + } + } + for _, n := range []int{16, 17} { + r := Result{Artifacts: make([]Artifact, n)} + for i := range r.Artifacts { + r.Artifacts[i] = Artifact{URL: "https://example.com", Label: "a"} + } + if e := validateResult(r); (e == nil) != (n == 16) { + t.Errorf("artifacts %d %v", n, e) + } + } + for _, n := range []int{256, 257} { + if e := validateArtifact(Artifact{URL: "https://example.com", Label: strings.Repeat("a", n)}); (e == nil) != (n == 256) { + t.Errorf("label %d %v", n, e) + } + } + for _, n := range []int{2048, 2049} { + u := "https://example.com/" + u += strings.Repeat("a", n-len(u)) + if e := validateArtifact(Artifact{URL: u}); (e == nil) != (n == 2048) { + t.Errorf("URL %d %v", n, e) + } + } + r := Result{Text: strings.Repeat("a", 262144-len("https://example.com")-256), Artifacts: []Artifact{{URL: "https://example.com", Label: strings.Repeat("a", 256)}}} + if e := validateResult(r); e != nil { + t.Fatal(e) + } + r.Text += "a" + if validateResult(r) == nil { + t.Error("artifact budget bypass") + } + b, e := seal(testKey, []byte("secret")) + if e != nil { + t.Fatal(e) + } + b2, e := seal(testKey, []byte("secret")) + if e != nil || bytes.Equal(b, b2) { + t.Error("nonce reused") + } + b[1] = 2 + if _, e = openEnvelope(testKey, b); e == nil { + t.Error("envelope version") + } + wrong := append([]byte{}, testKey...) + wrong[0] ^= 1 + if _, e = openEnvelope(wrong, b2); e == nil { + t.Error("wrong key accepted") + } +} +func TestContractConfigAndPaths(t *testing.T) { + s, dir := openTestStore(t) + if e := s.SetEnabled(false); e != nil { + t.Fatal(e) + } + cfg, e := reopen(t, dir).ConfigStatus() + if e != nil || cfg.Enabled { + t.Errorf("config %v %#v", e, cfg) + } + for _, path := range []string{"relative", filepath.VolumeName(dir) + string(filepath.Separator), dir + string(filepath.Separator) + ".."} { + if _, e := Open(path, testKey); e == nil { + t.Errorf("unsafe dir %q", path) + } + } + for _, n := range []int{0, 31, 33} { + if _, e := Open(dir, make([]byte, n)); e == nil { + t.Error("bad key accepted") + } + } +} +func TestContractUnknownCancelIntent(t *testing.T) { + s, _ := openTestStore(t) + j := mustSubmit(t, s, "unknown-cancel") + c, e := s.Claim("client.a", j.ID, "worker") + if e != nil { + t.Fatal(e) + } + if e = s.Recover(); e != nil { + t.Fatal(e) + } + j, e = s.Cancel("client.a", j.ID) + if e != nil || !j.CancelRequested || j.State != "outcome_unknown" { + t.Fatalf("unknown cancel intent: %v %#v", e, j) + } + if _, e = s.Complete(j.ID, c.Token, Result{Cancelled: true}); e != nil { + t.Fatal(e) + } +} +func TestContractOwnFileHardlinks(t *testing.T) { + for _, name := range []string{lockName, stateFileName} { + t.Run(name, func(t *testing.T) { + s, dir := openTestStore(t) + j := mustSubmit(t, s, "hardlink") + if e := os.Link(filepath.Join(dir, name), filepath.Join(dir, "alias")); e != nil { + t.Fatal(e) + } + if _, e := s.Get("client.a", j.ID); e == nil { + t.Error("hardlink accepted") + } + }) + } +} +func TestContractDuplicateKeys(t *testing.T) { + s, dir := openTestStore(t) + j := mustSubmit(t, s, "duplicate-json") + b, _ := os.ReadFile(filepath.Join(dir, stateFileName)) + p, _ := openEnvelope(testKey, b) + var m map[string]json.RawMessage + if e := json.Unmarshal(p, &m); e != nil { + t.Fatal(e) + } + p = append([]byte(`{"enabled":true,`), p[1:]...) + b, _ = seal(testKey, p) + os.WriteFile(filepath.Join(dir, stateFileName), b, 0600) + if _, e := s.Get("client.a", j.ID); e == nil { + t.Error("duplicate keys accepted") + } +} diff --git a/internal/dotqueue/crypto.go b/internal/dotqueue/crypto.go index d4327ce..6baa577 100644 --- a/internal/dotqueue/crypto.go +++ b/internal/dotqueue/crypto.go @@ -25,7 +25,7 @@ func seal(key []byte, plaintext []byte) ([]byte, error) { if _, err := io.ReadFull(rand.Reader, nonce); err != nil { return nil, errf("unknown") } - ciphertext := gcm.Seal(nil, nonce, plaintext, nil) + ciphertext := gcm.Seal(nil, nonce, plaintext, []byte{'D', envelopeVersion}) out := make([]byte, 2+len(nonce)+len(ciphertext)) out[0] = 'D' out[1] = envelopeVersion @@ -54,7 +54,7 @@ func openEnvelope(key []byte, data []byte) ([]byte, error) { } nonce := data[2 : 2+gcm.NonceSize()] ciphertext := data[2+gcm.NonceSize():] - plain, err := gcm.Open(nil, nonce, ciphertext, nil) + plain, err := gcm.Open(nil, nonce, ciphertext, data[:2]) if err != nil { return nil, errf("corrupt") } diff --git a/internal/dotqueue/files_unix.go b/internal/dotqueue/files_unix.go index c8f18c0..13c75e8 100644 --- a/internal/dotqueue/files_unix.go +++ b/internal/dotqueue/files_unix.go @@ -3,6 +3,7 @@ package dotqueue import ( + "golang.org/x/sys/unix" "os" "syscall" ) @@ -30,6 +31,14 @@ func rejectHardlink(f *os.File, info os.FileInfo) error { return nil } +func tryHandleLock(f *os.File) (bool, error) { + e := unix.Flock(int(f.Fd()), unix.LOCK_EX|unix.LOCK_NB) + if e == unix.EWOULDBLOCK || e == unix.EAGAIN { + return false, nil + } + return e == nil, e +} +func unlockHandle(f *os.File) { _ = unix.Flock(int(f.Fd()), unix.LOCK_UN) } func replaceFile(from, to string) error { return os.Rename(from, to) } func syncDirectory(path string) error { diff --git a/internal/dotqueue/files_unix_test.go b/internal/dotqueue/files_unix_test.go new file mode 100644 index 0000000..e3c6f5d --- /dev/null +++ b/internal/dotqueue/files_unix_test.go @@ -0,0 +1,45 @@ +//go:build !windows + +package dotqueue + +import ( + "os" + "path/filepath" + "testing" +) + +func TestUnixPrivacyAndSymlinks(t *testing.T) { + s, dir := openTestStore(t) + j := mustSubmit(t, s, "privacy") + for _, p := range []string{dir, filepath.Join(dir, markerName), filepath.Join(dir, stateFileName), filepath.Join(dir, lockName)} { + if e := os.Chmod(p, 0777); e != nil { + t.Fatal(e) + } + } + if _, e := s.Get("client.a", j.ID); e != nil { + t.Fatal(e) + } + for _, p := range []string{dir, filepath.Join(dir, markerName), filepath.Join(dir, stateFileName), filepath.Join(dir, lockName)} { + i, e := os.Stat(p) + if e != nil { + t.Fatal(e) + } + want := os.FileMode(0600) + if p == dir { + want = 0700 + } + if i.Mode().Perm() != want { + t.Error(p, i.Mode()) + } + } + marker := filepath.Join(dir, markerName) + os.Remove(marker) + target := filepath.Join(t.TempDir(), "marker") + os.WriteFile(target, []byte("1\n"), 0600) + if e := os.Symlink(target, marker); e != nil { + t.Fatal(e) + } + if _, e := s.Get("client.a", j.ID); e == nil { + t.Error("symlink marker accepted") + } +} diff --git a/internal/dotqueue/files_windows.go b/internal/dotqueue/files_windows.go index c68a9f1..bff53cb 100644 --- a/internal/dotqueue/files_windows.go +++ b/internal/dotqueue/files_windows.go @@ -5,6 +5,7 @@ package dotqueue import ( "os" "syscall" + "time" "golang.org/x/sys/windows" ) @@ -59,7 +60,29 @@ func replaceFile(from, to string) error { if err != nil { return err } - return windows.MoveFileEx(source, target, windows.MOVEFILE_REPLACE_EXISTING|windows.MOVEFILE_WRITE_THROUGH) + // Windows scanners/readers can temporarily deny replacement even after our + // own handles close. Retry only these transient errors, under the queue lock. + deadline := time.Now().Add(time.Second) + for { + e := windows.MoveFileEx(source, target, windows.MOVEFILE_REPLACE_EXISTING|windows.MOVEFILE_WRITE_THROUGH) + if e == nil || e != windows.ERROR_ACCESS_DENIED && e != windows.ERROR_SHARING_VIOLATION || time.Now().After(deadline) { + return e + } + time.Sleep(10 * time.Millisecond) + } } func syncDirectory(string) error { return nil } + +func tryHandleLock(f *os.File) (bool, error) { + var o windows.Overlapped + e := windows.LockFileEx(windows.Handle(f.Fd()), windows.LOCKFILE_EXCLUSIVE_LOCK|windows.LOCKFILE_FAIL_IMMEDIATELY, 0, 1, 0, &o) + if e == windows.ERROR_LOCK_VIOLATION { + return false, nil + } + return e == nil, e +} +func unlockHandle(f *os.File) { + var o windows.Overlapped + _ = windows.UnlockFileEx(windows.Handle(f.Fd()), 0, 1, 0, &o) +} diff --git a/internal/dotqueue/files_windows_test.go b/internal/dotqueue/files_windows_test.go new file mode 100644 index 0000000..61c7bcd --- /dev/null +++ b/internal/dotqueue/files_windows_test.go @@ -0,0 +1,87 @@ +//go:build windows + +package dotqueue + +import ( + "golang.org/x/sys/windows" + "os" + "os/exec" + "path/filepath" + "runtime" + "testing" + "unsafe" +) + +func TestWindowsPrivacyRepairedOnRead(t *testing.T) { + s, dir := openTestStore(t) + j := mustSubmit(t, s, "acl") + user, e := windows.GetCurrentProcessToken().GetTokenUser() + if e != nil { + t.Fatal(e) + } + paths := []string{dir, filepath.Join(dir, stateFileName), filepath.Join(dir, markerName), filepath.Join(dir, lockName)} + for _, p := range paths { + sd, e := windows.SecurityDescriptorFromString("D:P(A;;FA;;;WD)") + if e != nil { + t.Fatal(e) + } + acl, _, e := sd.DACL() + if e != nil { + t.Fatal(e) + } + if e = windows.SetNamedSecurityInfo(p, windows.SE_FILE_OBJECT, windows.DACL_SECURITY_INFORMATION|windows.PROTECTED_DACL_SECURITY_INFORMATION, nil, nil, acl, nil); e != nil { + t.Fatal(e) + } + } + if _, e = s.Get("client.a", j.ID); e != nil { + t.Fatal(e) + } + for _, p := range paths { + sd, e := windows.GetNamedSecurityInfo(p, windows.SE_FILE_OBJECT, windows.DACL_SECURITY_INFORMATION) + if e != nil { + t.Fatal(e) + } + flags, _, e := sd.Control() + if e != nil || flags&windows.SE_DACL_PROTECTED == 0 { + t.Fatalf("unprotected ACL %s %v", p, e) + } + acl, _, e := sd.DACL() + if e != nil || acl == nil || acl.AceCount != 1 { + t.Fatalf("ACL count %s %v", p, e) + } + var ace *windows.ACCESS_ALLOWED_ACE + if e = windows.GetAce(acl, 0, &ace); e != nil { + t.Fatal(e) + } + sid := (*windows.SID)(unsafe.Pointer(&ace.SidStart)) + if !sid.IsValid() || !sid.Equals(user.User.Sid) || ace.Header.AceType != windows.ACCESS_ALLOWED_ACE_TYPE || ace.Mask != windows.STANDARD_RIGHTS_REQUIRED|windows.SYNCHRONIZE|0x1ff { + t.Fatalf("ACL principal/rights %s", p) + } + runtime.KeepAlive(sd) + } +} +func TestWindowsReparseAndLocalPaths(t *testing.T) { + s, dir := openTestStore(t) + j := mustSubmit(t, s, "junction") + marker := filepath.Join(dir, markerName) + if e := os.Remove(marker); e != nil { + t.Fatal(e) + } + target := t.TempDir() + cmd := exec.Command("cmd.exe", "/c", "mklink", "/J", marker, target) + if b, e := cmd.CombinedOutput(); e != nil { + t.Fatalf("junction fixture: %v %s", e, b) + } + defer os.Remove(marker) + if _, e := s.Get("client.a", j.ID); e == nil { + t.Error("marker reparse accepted") + } + if _, e := Open(filepath.Join(marker, "child"), testKey); e == nil { + t.Error("reparse ancestor accepted") + } + for _, p := range []string{`\\server\share\queue`, `\\?\C:\queue`, `C:\queue:stream`, `C:relative`} { + if _, e := Open(p, testKey); e == nil { + t.Errorf("nonlocal path accepted %q", p) + } + } +} diff --git a/internal/dotqueue/paths.go b/internal/dotqueue/paths.go new file mode 100644 index 0000000..2e5d29c --- /dev/null +++ b/internal/dotqueue/paths.go @@ -0,0 +1,75 @@ +package dotqueue + +import ( + "io" + "os" + "path/filepath" + "strings" + "unicode" +) + +// Paths must be canonical local absolute non-root directories. Ancestors must +// be real directories; the caller supplies trusted parents owned/administered +// by the local user or OS. This is not isolation from a malicious same-user process. +func localDir(dir string) bool { + if !filepath.IsAbs(dir) || len(dir) > 32768 || filepath.Clean(dir) != dir || strings.ContainsFunc(dir, unicode.IsControl) { + return false + } + vol := filepath.VolumeName(dir) + if strings.HasPrefix(vol, `\\`) || dir == filepath.VolumeName(dir)+string(filepath.Separator) { + return false + } + rest := strings.TrimPrefix(dir, vol) + if strings.Contains(rest, ":") { + return false + } + return true +} +func checkAncestors(dir string) error { + for p := dir; ; p = filepath.Dir(p) { + i, e := os.Lstat(p) + if e == nil { + if !i.IsDir() || rejectLink(i) != nil { + return errf("unknown") + } + } else if !os.IsNotExist(e) { + return errf("unknown") + } + if filepath.Dir(p) == p { + break + } + } + return nil +} +func (s *Store) readPrivate(path string, limit int64) ([]byte, error) { + if regularFile(path) != nil { + return nil, errf("corrupt") + } + f, e := os.Open(path) + if e != nil { + return nil, errf("corrupt") + } + defer f.Close() + i, e := f.Stat() + if e != nil || !i.Mode().IsRegular() || rejectHardlink(f, i) != nil || i.Size() > limit { + return nil, errf("corrupt") + } + if privatePath(path, false) != nil { + return nil, errf("unknown") + } + b, e := io.ReadAll(io.LimitReader(f, limit+1)) + if e != nil || int64(len(b)) > limit { + return nil, errf("corrupt") + } + return b, nil +} + +type ConfigStatus struct { + Enabled bool `json:"enabled"` +} + +func (s *Store) ConfigStatus() (ConfigStatus, error) { + var out ConfigStatus + e := s.withLock(false, false, func(st *persistenceState) error { out.Enabled = st.Enabled; return nil }) + return out, e +} diff --git a/internal/dotqueue/review_test.go b/internal/dotqueue/review_test.go new file mode 100644 index 0000000..0b237bc --- /dev/null +++ b/internal/dotqueue/review_test.go @@ -0,0 +1,364 @@ +package dotqueue + +import ( + "bytes" + "encoding/json" + "fmt" + "os" + "os/exec" + "path/filepath" + "reflect" + "strings" + "sync" + "testing" +) + +func mustSubmit(t *testing.T, s *Store, id string) Job { + t.Helper() + j, e := s.Submit("client.a", sampleSubmission(id)) + if e != nil { + t.Fatal(e) + } + return j +} +func reopen(t *testing.T, dir string) *Store { + t.Helper() + s, e := Open(dir, testKey) + if e != nil { + t.Fatal(e) + } + return s +} +func TestReviewTokensAndUnclaimedComplete(t *testing.T) { + s, _ := openTestStore(t) + j := mustSubmit(t, s, "token") + for _, token := range []string{"", "x", strings.Repeat("a", 64)} { + if _, e := s.Complete(j.ID, token, Result{Text: "forged"}); e == nil { + t.Errorf("unclaimed Complete accepted token %q", token) + } + } + got, e := s.Get("client.a", j.ID) + if e != nil || got.State != "queued" { + t.Fatalf("unclaimed job changed: %v %#v", e, got) + } + c, e := s.Claim("client.a", j.ID, "worker") + if e != nil { + t.Fatal(e) + } + for _, token := range []string{"", "x", strings.Repeat("f", 64), strings.ToUpper(c.Token)} { + if token == c.Token { + continue + } + if _, e := s.Progress(j.ID, token, "running", "bad"); e == nil { + t.Error("wrong Progress token accepted") + } + if _, e := s.Complete(j.ID, token, Result{}); e == nil { + t.Error("wrong Complete token accepted") + } + } +} +func TestReviewUnknownReconcileAndSummary(t *testing.T) { + for _, cancel := range []bool{false, true} { + t.Run(fmt.Sprint(cancel), func(t *testing.T) { + s, dir := openTestStore(t) + j := mustSubmit(t, s, "recover") + c, e := s.Claim("client.a", j.ID, "worker") + if e != nil { + t.Fatal(e) + } + p, e := s.Progress(j.ID, c.Token, "claimed", "durable summary") + if e != nil { + t.Error(e) + } + v := reflect.ValueOf(p).FieldByName("Summary") + if !v.IsValid() || v.String() != "durable summary" { + t.Error("summary not observable") + } + s = reopen(t, dir) + g, e := s.Get("client.a", j.ID) + if e != nil { + t.Fatal(e) + } + v = reflect.ValueOf(g).FieldByName("Summary") + if !v.IsValid() || v.String() != "durable summary" { + t.Error("summary not persisted") + } + if cancel { + if _, e = s.Cancel("client.a", j.ID); e != nil { + t.Fatal(e) + } + } + if e = s.Recover(); e != nil { + t.Fatal(e) + } + before, _ := s.Get("client.a", j.ID) + if e = s.Recover(); e != nil { + t.Fatal(e) + } + after, _ := s.Get("client.a", j.ID) + if !before.UpdatedAt.Equal(after.UpdatedAt) { + t.Error("recovery repeated") + } + if _, e = s.Claim("client.a", j.ID, "new"); e == nil { + t.Error("unknown reclaimed") + } + r := Result{Text: "authoritative secret result", Cancelled: cancel} + done, e := s.Complete(j.ID, c.Token, r) + if e != nil { + t.Fatalf("former token reconciliation: %v", e) + } + want := "completed" + if cancel { + want = "cancelled" + } + if done.State != want { + t.Error(done.State) + } + s = reopen(t, dir) + done, e = s.Get("client.a", j.ID) + if e != nil || done.Result == nil || done.Result.Text != r.Text { + t.Fatalf("terminal persistence: %v %#v", e, done) + } + if _, e = s.Complete(j.ID, c.Token, r); e != nil { + t.Error(e) + } + r.Text = "different" + if _, e = s.Complete(j.ID, c.Token, r); e == nil { + t.Error("conflicting terminal retry") + } + entries, _ := os.ReadDir(dir) + for _, entry := range entries { + b, _ := os.ReadFile(filepath.Join(dir, entry.Name())) + for _, secret := range []string{"hello dot", "durable summary", "authoritative secret result", c.Token} { + if bytes.Contains(b, []byte(secret)) { + t.Errorf("plaintext %s", entry.Name()) + } + } + } + }) + } +} +func TestReviewDisabledAllOperations(t *testing.T) { + s, dir := openTestStore(t) + j := mustSubmit(t, s, "disabled-all") + c, e := s.Claim("client.a", j.ID, "worker") + if e != nil { + t.Fatal(e) + } + if e = s.SetEnabled(false); e != nil { + t.Fatal(e) + } + s = reopen(t, dir) + ops := map[string]func() error{"Get": func() error { _, e := s.Get("client.a", j.ID); return e }, "ByRequest": func() error { _, e := s.ByRequest("client.a", j.RequestID); return e }, "Pending": func() error { _, e := s.Pending("client.a"); return e }, "Submit": func() error { _, e := s.Submit("client.a", sampleSubmission("new")); return e }, "Claim": func() error { _, e := s.Claim("client.a", j.ID, "worker"); return e }, "Progress": func() error { _, e := s.Progress(j.ID, c.Token, "running", ""); return e }, "Complete": func() error { _, e := s.Complete(j.ID, c.Token, Result{}); return e }, "Cancel": func() error { _, e := s.Cancel("client.a", j.ID); return e }, "Recover": s.Recover} + for name, op := range ops { + t.Run(name, func(t *testing.T) { + e := op() + if e == nil || e.Error() != "dot_queue_disabled" { + t.Fatalf("want disabled got %v", e) + } + }) + } +} +func TestReviewLiteralBounds(t *testing.T) { + for _, n := range []int{262144, 262145} { + t.Run(fmt.Sprint(n), func(t *testing.T) { + s, _ := openTestStore(t) + sub := Submission{RequestID: "bound", Mode: "task", Task: strings.Repeat("t", n-1), Messages: []Message{{Role: "user", Content: "m"}}} + _, e := s.Submit("client.a", sub) + if (e == nil) != (n == 262144) { + t.Errorf("combined prompt %d: %v", n, e) + } + r := Result{Text: strings.Repeat("r", n-1), ErrorCode: "e"} + if e = validateResult(r); (e == nil) != (n == 262144) { + t.Errorf("total result %d: %v", n, e) + } + }) + } + for _, sub := range []Submission{{RequestID: "bad", Mode: "task", Task: string([]byte{255})}, {RequestID: "bad", Mode: "answer", Messages: []Message{{Role: "user", Content: string([]byte{255})}}}, {RequestID: "bad", Mode: "invalid"}, {RequestID: "bad", Mode: "answer"}, {RequestID: "bad", Mode: "answer", Messages: []Message{{Role: "tool", Content: "x"}}}} { + s, _ := openTestStore(t) + if _, e := s.Submit("client.a", sub); e == nil { + t.Errorf("bad submission accepted %#v", sub) + } + } + for _, raw := range []string{"https://localhost/a", "https://foo.localhost/a", "https://127.0.0.1/a", "https://10.0.0.1/a", "https://[::1]/a", "https://[fc00::1]/a", "https://169.254.1.1/a", "https://user:pass@example.com/a", "https://example.com/" + strings.Repeat("a", 2049), string([]byte{255})} { + if e := validateResult(Result{Artifacts: []Artifact{{URL: raw}}}); e == nil { + t.Errorf("bad artifact accepted %q", raw) + } + } + for _, r := range []Result{{ErrorCode: strings.Repeat("e", 129)}, {ErrorCode: "contains spaces"}, {ErrorCode: string([]byte{255})}, {Text: string([]byte{255})}, {Artifacts: make([]Artifact, 17)}, {Artifacts: []Artifact{{URL: "https://example.com", Label: strings.Repeat("a", 257)}}}} { + if e := validateResult(r); e == nil { + t.Errorf("bad result accepted %#v", r) + } + } + for _, n := range []int{128, 129} { + s, _ := openTestStore(t) + _, e := s.Submit(strings.Repeat("c", n), sampleSubmission(strings.Repeat("r", n))) + if (e == nil) != (n == 128) { + t.Errorf("ID length %d %v", n, e) + } + } + for _, n := range []int{2048, 2049} { + if e := validateSummary(strings.Repeat("s", n)); (e == nil) != (n == 2048) { + t.Errorf("summary %d %v", n, e) + } + } +} +func authenticatedFixture(t *testing.T, s *Store, mutate func(map[string]any)) { + t.Helper() + b, e := os.ReadFile(filepath.Join(s.dir, stateFileName)) + if e != nil { + t.Fatal(e) + } + p, e := openEnvelope(testKey, b) + if e != nil { + t.Fatal(e) + } + var obj map[string]any + if e = json.Unmarshal(p, &obj); e != nil { + t.Fatal(e) + } + mutate(obj) + p, e = json.Marshal(obj) + if e != nil { + t.Fatal(e) + } + b, e = seal(testKey, p) + if e != nil { + t.Fatal(e) + } + if e = os.WriteFile(filepath.Join(s.dir, stateFileName), b, 0600); e != nil { + t.Fatal(e) + } +} +func TestReviewAuthenticatedInvalidState(t *testing.T) { + cases := map[string]func(map[string]any){"enabled-null": func(m map[string]any) { m["enabled"] = nil }, "enabled-missing": func(m map[string]any) { delete(m, "enabled") }, "version": func(m map[string]any) { m["state_version"] = 2 }, "index-missing": func(m map[string]any) { m["request_index"] = map[string]any{} }} + jobCases := map[string]func(map[string]any){"nested-unknown": func(j map[string]any) { j["future"] = true }, "state": func(j map[string]any) { j["state"] = "bogus" }, "missing-state": func(j map[string]any) { delete(j, "state") }, "null-state": func(j map[string]any) { j["state"] = nil }, "wrong-type": func(j map[string]any) { j["cancel_requested"] = "false" }, "digest": func(j map[string]any) { j["request_digest"] = strings.Repeat("a", 64) }, "claim": func(j map[string]any) { j["state"] = "claimed" }, "terminal": func(j map[string]any) { j["state"] = "completed" }, "message-unknown": func(j map[string]any) { j["messages"].([]any)[0].(map[string]any)["extra"] = true }, "message-null": func(j map[string]any) { j["messages"].([]any)[0].(map[string]any)["content"] = nil }} + for name, mut := range jobCases { + mut := mut + cases[name] = func(m map[string]any) { + for _, j := range m["jobs"].(map[string]any) { + mut(j.(map[string]any)) + } + } + } + for name, mut := range cases { + t.Run(name, func(t *testing.T) { + s, _ := openTestStore(t) + j := mustSubmit(t, s, "fixture") + authenticatedFixture(t, s, mut) + if _, e := s.Get("client.a", j.ID); e == nil || e.Error() != "dot_queue_corrupt" { + t.Fatalf("invalid authenticated state accepted: %v", e) + } + }) + } +} +func TestReviewMarkerAndAuthentication(t *testing.T) { + for _, kind := range []string{"content", "version", "directory", "hardlink", "missing-state"} { + t.Run(kind, func(t *testing.T) { + s, dir := openTestStore(t) + j := mustSubmit(t, s, "marker") + path := filepath.Join(dir, markerName) + switch kind { + case "content": + os.WriteFile(path, []byte("garbage"), 0600) + case "version": + os.WriteFile(path, []byte("2\n"), 0600) + case "directory": + os.Remove(path) + os.Mkdir(path, 0700) + case "hardlink": + if e := os.Link(path, filepath.Join(dir, "alias")); e != nil { + t.Fatal(e) + } + case "missing-state": + os.Remove(filepath.Join(dir, stateFileName)) + if e := s.Initialize(true); e == nil { + t.Error("missing state reinitialized") + } + } + if _, e := s.Get("client.a", j.ID); e == nil { + t.Error("invalid marker accepted") + } + }) + } + s, dir := openTestStore(t) + j := mustSubmit(t, s, "auth") + b, _ := os.ReadFile(filepath.Join(dir, stateFileName)) + b[len(b)-1] ^= 1 + os.WriteFile(filepath.Join(dir, stateFileName), b, 0600) + if _, e := s.Get("client.a", j.ID); e == nil { + t.Error("tampered ciphertext accepted") + } +} + +// Child processes use only temporary fixture state and a fixed, non-secret test key. +func TestQueueProcessHelper(t *testing.T) { + dir := os.Getenv("DOTQUEUE_TEST_DIR") + if dir == "" { + return + } + s := reopen(t, dir) + switch os.Getenv("DOTQUEUE_TEST_OP") { + case "submit": + j, e := s.Submit("client.a", sampleSubmission("process-same")) + if e != nil { + t.Fatal(e) + } + fmt.Println("JOB=" + j.ID) + case "claim": + _, e := s.Claim("client.a", os.Getenv("DOTQUEUE_TEST_JOB"), os.Getenv("DOTQUEUE_TEST_WORKER")) + if e != nil { + fmt.Println("CLAIM=" + e.Error()) + } else { + fmt.Println("CLAIM=ok") + } + } +} +func TestReviewCrossProcessCAS(t *testing.T) { + s, dir := openTestStore(t) + run := func(op, job string) []string { + outs := make([]string, 2) + var wg sync.WaitGroup + start := make(chan struct{}) + for i := 0; i < 2; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + cmd := exec.Command(os.Args[0], "-test.run=^TestQueueProcessHelper$", "-test.v") + cmd.Env = append(os.Environ(), "DOTQUEUE_TEST_DIR="+dir, "DOTQUEUE_TEST_OP="+op, "DOTQUEUE_TEST_JOB="+job, fmt.Sprintf("DOTQUEUE_TEST_WORKER=worker.%d", i)) + <-start + b, e := cmd.CombinedOutput() + if e != nil { + t.Errorf("child: %v %s", e, b) + } + outs[i] = string(b) + }(i) + } + close(start) + wg.Wait() + return outs + } + submits := run("submit", "") + j, e := s.ByRequest("client.a", "process-same") + if e != nil { + t.Fatal(e) + } + for _, out := range submits { + if !strings.Contains(out, "JOB="+j.ID+"\n") && !strings.Contains(out, "JOB="+j.ID+"\r\n") { + t.Fatalf("child returned another job: %s", out) + } + } + ns, e := s.Pending("client.a") + if e != nil || len(ns) != 1 { + t.Fatalf("duplicate job/notice %v %v", ns, e) + } + ns2, _ := reopen(t, dir).Pending("client.a") + if !reflect.DeepEqual(ns, ns2) { + t.Error("notice changed on restart") + } + outs := run("claim", j.ID) + all := strings.Join(outs, "\n") + if strings.Count(all, "CLAIM=ok") != 1 || strings.Count(all, "CLAIM=dot_queue_claim_conflict") != 1 { + t.Fatalf("competing claim: %s", all) + } +} diff --git a/internal/dotqueue/schema.go b/internal/dotqueue/schema.go new file mode 100644 index 0000000..58a231c --- /dev/null +++ b/internal/dotqueue/schema.go @@ -0,0 +1,79 @@ +package dotqueue + +import ( + "bytes" + "encoding/json" + "reflect" + "strings" + "time" +) + +// strictShape validates required fields, nullability and exact nested schemas +// before encoding/json can turn missing/null scalars into usable zero values. +func strictShape(raw []byte, t reflect.Type) error { + if t.Kind() == reflect.Pointer { + if bytes.Equal(raw, []byte("null")) { + return errf("corrupt") + } + return strictShape(raw, t.Elem()) + } + if bytes.Equal(raw, []byte("null")) { + return errf("corrupt") + } + if t == reflect.TypeOf(time.Time{}) { + var v time.Time + if json.Unmarshal(raw, &v) != nil { + return errf("corrupt") + } + return nil + } + switch t.Kind() { + case reflect.Struct: + var m map[string]json.RawMessage + if json.Unmarshal(raw, &m) != nil || m == nil { + return errf("corrupt") + } + fields := map[string]reflect.StructField{} + for i := 0; i < t.NumField(); i++ { + f := t.Field(i) + tag := strings.Split(f.Tag.Get("json"), ",") + fields[tag[0]] = f + if _, ok := m[tag[0]]; !ok && !strings.Contains(f.Tag.Get("json"), "omitempty") { + return errf("corrupt") + } + } + for k, v := range m { + f, ok := fields[k] + if !ok || strictShape(v, f.Type) != nil { + return errf("corrupt") + } + } + case reflect.Map: + var m map[string]json.RawMessage + if json.Unmarshal(raw, &m) != nil || m == nil { + return errf("corrupt") + } + for _, v := range m { + if strictShape(v, t.Elem()) != nil { + return errf("corrupt") + } + } + case reflect.Slice: + // Nil message/artifact lists are canonicalized to [] before persistence. + var a []json.RawMessage + if json.Unmarshal(raw, &a) != nil || a == nil { + return errf("corrupt") + } + for _, v := range a { + if strictShape(v, t.Elem()) != nil { + return errf("corrupt") + } + } + default: + v := reflect.New(t).Interface() + if json.Unmarshal(raw, v) != nil { + return errf("corrupt") + } + } + return nil +} diff --git a/internal/dotqueue/state.go b/internal/dotqueue/state.go index 1e33a49..06f9b2b 100644 --- a/internal/dotqueue/state.go +++ b/internal/dotqueue/state.go @@ -2,19 +2,21 @@ package dotqueue import ( "encoding/json" + "reflect" "time" ) const ( - stateVersion = 1 - markerName = "dot_queue.initialized" - lockName = "dot_queue.lock" - stateFileName = "queue.state.dq" - maxStateBytes = 64 * 1024 * 1024 - claimTokenBytes = 32 + stateVersion = 1 + markerName = "dot_queue.initialized" + lockName = "dot_queue.lock" + stateFileName = "queue.state.dq" + maxStateBytes = 64 * 1024 * 1024 + claimTokenBytes = 32 ) type jobRecord struct { + Summary string `json:"summary"` ID string `json:"id"` RequestID string `json:"request_id"` RequestDigest string `json:"request_digest"` @@ -38,11 +40,11 @@ type jobRecord struct { } type persistenceState struct { - StateVersion int `json:"state_version"` - Enabled bool `json:"enabled"` - Jobs map[string]*jobRecord `json:"jobs"` - RequestIndex map[string]string `json:"request_index"` - Order []string `json:"order"` + StateVersion int `json:"state_version"` + Enabled bool `json:"enabled"` + Jobs map[string]*jobRecord `json:"jobs"` + RequestIndex map[string]string `json:"request_index"` + Order []string `json:"order"` } func newPersistenceState(enabled bool) *persistenceState { @@ -73,6 +75,9 @@ func parsePersistenceState(data []byte) (*persistenceState, error) { return nil, errf("corrupt") } } + if strictShape(data, reflect.TypeOf(persistenceState{})) != nil { + return nil, errf("corrupt") + } var st persistenceState if json.Unmarshal(data, &st) != nil { return nil, errf("corrupt") @@ -83,7 +88,24 @@ func parsePersistenceState(data []byte) (*persistenceState, error) { if st.Jobs == nil || st.RequestIndex == nil || st.Order == nil { return nil, errf("corrupt") } + if len(st.Jobs) > 100 || len(st.RequestIndex) != len(st.Jobs) { + return nil, errf("corrupt") + } + events := map[string]bool{} + tokens := map[string]bool{} for id, job := range st.Jobs { + if job != nil { + if events[job.EventID] || job.ClaimToken != "" && tokens[job.ClaimToken] { + return nil, errf("corrupt") + } + events[job.EventID] = true + if job.ClaimToken != "" { + tokens[job.ClaimToken] = true + } + } + if job != nil && st.RequestIndex[requestKey(job.ClientID, job.RequestID)] != id { + return nil, errf("corrupt") + } if job == nil || job.ID != id { return nil, errf("corrupt") } @@ -129,6 +151,65 @@ func validateStoredJob(job *jobRecord) error { if err := validateID(job.EventID); err != nil { return errf("corrupt") } + if validateSubmission(submissionFromRecord(job)) != nil || validateSummary(job.Summary) != nil || job.CreatedAt.IsZero() || job.UpdatedAt.Before(job.CreatedAt) { + return errf("corrupt") + } + digest, e := submissionDigest(submissionFromRecord(job)) + if e != nil || digest != job.RequestDigest { + return errf("corrupt") + } + claimed := job.ClaimToken != "" || job.WorkerID != "" + if claimed && (!validHex(job.ClaimToken) || validateID(job.WorkerID) != nil) { + return errf("corrupt") + } + if job.Recovered && !claimed { + return errf("corrupt") + } + if job.Result == nil { + if job.ResultDigest != "" || job.ErrorCode != "" { + return errf("corrupt") + } + } else { + if validateResult(*job.Result) != nil || !claimed { + return errf("corrupt") + } + d, e := resultDigest(*job.Result) + if e != nil || d != job.ResultDigest || job.ErrorCode != job.Result.ErrorCode { + return errf("corrupt") + } + } + switch job.State { + case StateQueued: + if claimed || job.CancelRequested || job.Result != nil || job.Summary != "" { + return errf("corrupt") + } + case StateClaimed, StateRunning, StateWaitingUser: + if !claimed || job.CancelRequested || job.Result != nil || job.Recovered { + return errf("corrupt") + } + case StateCancelRequested: + if !claimed || !job.CancelRequested || job.Result != nil || job.Recovered { + return errf("corrupt") + } + case StateOutcomeUnknown: + if !claimed || !job.Recovered || job.Result != nil { + return errf("corrupt") + } + case StateCompleted: + if job.Result == nil || job.Result.Cancelled || job.Result.ErrorCode != "" { + return errf("corrupt") + } + case StateFailed: + if job.Result == nil || job.Result.Cancelled || job.Result.ErrorCode == "" { + return errf("corrupt") + } + case StateCancelled: + if !job.CancelRequested || claimed && (job.Result == nil || !job.Result.Cancelled) || !claimed && job.Result != nil { + return errf("corrupt") + } + default: + return errf("corrupt") + } return nil } @@ -150,6 +231,7 @@ func splitRequestKey(key string) [2]string { func publicJob(rec *jobRecord) Job { return Job{ + Summary: rec.Summary, ID: rec.ID, RequestID: rec.RequestID, RequestDigest: rec.RequestDigest, diff --git a/internal/dotqueue/store.go b/internal/dotqueue/store.go index 9ba98ec..8c400fb 100644 --- a/internal/dotqueue/store.go +++ b/internal/dotqueue/store.go @@ -1,18 +1,13 @@ package dotqueue import ( - "context" "crypto/rand" "encoding/hex" "encoding/json" "io" "os" "path/filepath" - "strings" "time" - "unicode" - - "github.com/gofrs/flock" ) type Store struct { @@ -24,7 +19,7 @@ func Open(dir string, key []byte) (*Store, error) { if len(key) != 32 { return nil, errf("unknown") } - if !filepath.IsAbs(dir) || len(dir) > 32768 || strings.ContainsFunc(dir, unicode.IsControl) { + if !localDir(dir) || checkAncestors(dir) != nil { return nil, errf("unknown") } return &Store{dir: dir, key: append([]byte(nil), key...)}, nil @@ -95,7 +90,7 @@ func (s *Store) Submit(clientID string, sub Submission) (Job, error) { State: StateQueued, SessionID: sub.SessionID, Task: sub.Task, - Messages: append([]Message(nil), sub.Messages...), + Messages: append([]Message{}, sub.Messages...), CreatedAt: now, UpdatedAt: now, EventID: eventID, @@ -118,6 +113,9 @@ func (s *Store) Get(clientID, jobID string) (Job, error) { } var out Job err := s.withLock(false, false, func(st *persistenceState) error { + if !st.Enabled { + return errf("disabled") + } rec, err := s.jobForClient(st, clientID, jobID) if err != nil { return err @@ -137,6 +135,9 @@ func (s *Store) ByRequest(clientID, requestID string) (Job, error) { } var out Job err := s.withLock(false, false, func(st *persistenceState) error { + if !st.Enabled { + return errf("disabled") + } jobID, ok := st.RequestIndex[requestKey(clientID, requestID)] if !ok { return errf("notfound") @@ -221,12 +222,13 @@ func (s *Store) Progress(jobID, token, state, summary string) (Job, error) { if rec == nil { return errf("notfound") } - if rec.ClaimToken != token { + if !validHex(token) || !validHex(rec.ClaimToken) || rec.ClaimToken != token { return errf("conflict") } switch rec.State { case StateClaimed, StateRunning, StateWaitingUser, StateCancelRequested: rec.State = state + rec.Summary = summary if rec.CancelRequested { rec.State = StateCancelRequested } @@ -260,16 +262,21 @@ func (s *Store) Complete(jobID, token string, result Result) (Job, error) { if rec == nil { return errf("notfound") } - if rec.ClaimToken != token { + if !validHex(token) || !validHex(rec.ClaimToken) || rec.ClaimToken != token { return errf("conflict") } - if terminalState(rec.State) { + if terminalState(rec.State) && rec.State != StateOutcomeUnknown { if rec.ResultDigest == digest { out = publicJob(rec) return nil } return errf("conflict") } + switch rec.State { + case StateClaimed, StateRunning, StateWaitingUser, StateCancelRequested, StateOutcomeUnknown: + default: + return errf("conflict") + } if result.Cancelled { if !rec.CancelRequested { return errf("conflict") @@ -283,7 +290,7 @@ func (s *Store) Complete(jobID, token string, result Result) (Job, error) { } rec.Result = &Result{ Text: result.Text, - Artifacts: append([]Artifact(nil), result.Artifacts...), + Artifacts: append([]Artifact{}, result.Artifacts...), ErrorCode: result.ErrorCode, Cancelled: result.Cancelled, } @@ -311,7 +318,7 @@ func (s *Store) Cancel(clientID, jobID string) (Job, error) { if err != nil { return err } - if terminalState(rec.State) { + if terminalState(rec.State) && rec.State != StateOutcomeUnknown { out = publicJob(rec) return nil } @@ -320,6 +327,8 @@ func (s *Store) Cancel(clientID, jobID string) (Job, error) { switch rec.State { case StateQueued: rec.State = StateCancelled + case StateOutcomeUnknown: + // Preserve uncertainty while recording new cancellation intent. default: rec.State = StateCancelRequested } @@ -357,6 +366,9 @@ func (s *Store) Pending(queueID string) ([]Notice, error) { } var out []Notice err := s.withLock(false, false, func(st *persistenceState) error { + if !st.Enabled { + return errf("disabled") + } for _, id := range st.Order { rec := st.Jobs[id] if rec == nil || rec.QueueID != queueID || rec.State != StateQueued { @@ -395,21 +407,34 @@ func (s *Store) withLock(allowCreate bool, mutate bool, fn func(*persistenceStat return errf("unknown") } info, statErr := f.Stat() - if statErr != nil || rejectHardlink(f, info) != nil { + if statErr != nil || !info.Mode().IsRegular() || rejectHardlink(f, info) != nil { f.Close() return errf("unknown") } - if f.Close() != nil || privatePath(lockPath, false) != nil { + defer f.Close() + if privatePath(lockPath, false) != nil { return errf("unknown") } - lock := flock.New(lockPath) - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - ok, err := lock.TryLockContext(ctx, 10*time.Millisecond) - if err != nil || !ok { + deadline := time.Now().Add(5 * time.Second) + for { + ok, e := tryHandleLock(f) + if e != nil { + return errf("unknown") + } + if ok { + break + } + if time.Now().After(deadline) { + return errf("unknown") + } + time.Sleep(10 * time.Millisecond) + } + defer unlockHandle(f) + // Verify the locked handle still names the checked private regular file. + current, e := os.Lstat(lockPath) + if e != nil || rejectLink(current) != nil || !os.SameFile(info, current) || rejectHardlink(f, info) != nil { return errf("unknown") } - defer lock.Unlock() st, err := s.loadState(allowCreate) if err != nil { @@ -448,6 +473,10 @@ func (s *Store) loadState(allowCreate bool) (*persistenceState, error) { } return nil, errf("corrupt") } + markerData, err := s.readPrivate(marker, 16) + if err != nil || string(markerData) != "1\n" { + return nil, errf("corrupt") + } if err := regularFile(statePath); err != nil { return nil, wrapUnknown(err) } @@ -460,6 +489,9 @@ func (s *Store) loadState(allowCreate bool) (*persistenceState, error) { if err != nil || !info.Mode().IsRegular() || rejectHardlink(f, info) != nil { return nil, errf("corrupt") } + if privatePath(statePath, false) != nil { + return nil, errf("unknown") + } if info.Size() > maxStateBytes { return nil, errf("corrupt") } @@ -524,7 +556,7 @@ func (s *Store) ensureMarker() error { return errf("unknown") } info, statErr := f.Stat() - if statErr != nil || rejectHardlink(f, info) != nil { + if statErr != nil || !info.Mode().IsRegular() || rejectHardlink(f, info) != nil { f.Close() return errf("unknown") } @@ -550,7 +582,7 @@ func (s *Store) ensureMarker() error { } func (s *Store) checkDir(create bool) error { - if !filepath.IsAbs(s.dir) || len(s.dir) > 32768 || strings.ContainsFunc(s.dir, unicode.IsControl) { + if !localDir(s.dir) || checkAncestors(s.dir) != nil { return errf("unknown") } if create { @@ -565,7 +597,7 @@ func (s *Store) checkDir(create bool) error { if err != nil || !info.IsDir() || rejectLink(info) != nil { return errf("unknown") } - if create && privatePath(s.dir, true) != nil { + if privatePath(s.dir, true) != nil { return errf("unknown") } return nil diff --git a/internal/dotqueue/store_test.go b/internal/dotqueue/store_test.go index ceebdd9..0ffb119 100644 --- a/internal/dotqueue/store_test.go +++ b/internal/dotqueue/store_test.go @@ -198,8 +198,8 @@ func TestDisabledRejectsSensitiveOps(t *testing.T) { t.Fatalf("claim should be disabled: %v", err) } _, err = store.Get("client.a", job.ID) - if err != nil { - t.Fatalf("read should work while disabled: %v", err) + if err == nil || err.Error() != "dot_queue_disabled" { + t.Fatalf("read should be disabled: %v", err) } } @@ -293,7 +293,7 @@ func TestConcurrentSubmitAndClaim(t *testing.T) { func TestMaxRetainedJobs(t *testing.T) { store, _ := openTestStore(t) - for i := 0; i < maxRetainedJobs; i++ { + for i := 0; i < 100; i++ { req := sampleSubmission(fmt.Sprintf("req.cap.%d", i)) if _, err := store.Submit("client.a", req); err != nil { t.Fatalf("fill %d: %v", i, err) @@ -304,4 +304,3 @@ func TestMaxRetainedJobs(t *testing.T) { t.Fatalf("expected capacity conflict, got %v", err) } } - diff --git a/internal/dotqueue/types.go b/internal/dotqueue/types.go index 58e162b..822a0b9 100644 --- a/internal/dotqueue/types.go +++ b/internal/dotqueue/types.go @@ -31,6 +31,7 @@ type Message struct { } type Job struct { + Summary string `json:"summary"` ID string `json:"id"` RequestID string `json:"request_id"` RequestDigest string `json:"request_digest"` diff --git a/internal/dotqueue/validate.go b/internal/dotqueue/validate.go index b67ff3f..0c366a6 100644 --- a/internal/dotqueue/validate.go +++ b/internal/dotqueue/validate.go @@ -1,30 +1,30 @@ package dotqueue import ( + "encoding/hex" + "net/netip" "net/url" "regexp" + "strconv" "strings" "unicode/utf8" ) const ( - maxMessages = 32 - maxPromptBytes = 256 * 1024 - maxResultBytes = 256 * 1024 - maxSummaryBytes = 2048 - maxArtifacts = 16 - maxLabelBytes = 256 - maxRetainedJobs = 100 - maxIDLen = 128 + maxMessages = 32 + maxPromptBytes = 256 * 1024 + maxResultBytes = 256 * 1024 + maxSummaryBytes = 2048 + maxArtifacts = 16 + maxLabelBytes = 256 + maxRetainedJobs = 100 + maxIDLen = 128 + maxURLBytes = 2048 ) var idPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$`) - -var allowedRoles = map[string]bool{ - "user": true, - "assistant": true, - "system": true, -} +var hostPattern = regexp.MustCompile(`^[a-zA-Z0-9](?:[a-zA-Z0-9.-]*[a-zA-Z0-9])?$`) +var allowedRoles = map[string]bool{"user": true, "assistant": true, "system": true} func validateID(id string) error { if !idPattern.MatchString(id) { @@ -32,88 +32,100 @@ func validateID(id string) error { } return nil } - -func validateSubmission(sub Submission) error { - if err := validateID(sub.RequestID); err != nil { - return err - } - if sub.Mode != ModeAnswer && sub.Mode != ModeTask { - return errf("unknown") +func validHex(s string) bool { + if len(s) != 64 || strings.ToLower(s) != s { + return false } - if sub.Mode == ModeAnswer && len(sub.Messages) == 0 { - return errf("unknown") - } - if sub.Mode == ModeTask && strings.TrimSpace(sub.Task) == "" { + _, e := hex.DecodeString(s) + return e == nil +} +func validateSubmission(s Submission) error { + if validateID(s.RequestID) != nil || (s.SessionID != "" && validateID(s.SessionID) != nil) || (s.Mode != ModeAnswer && s.Mode != ModeTask) || len(s.Messages) > maxMessages || !utf8.ValidString(s.Task) { return errf("unknown") } - if len(sub.Messages) > maxMessages { + if s.Mode == ModeAnswer && len(s.Messages) == 0 || s.Mode == ModeTask && strings.TrimSpace(s.Task) == "" { return errf("unknown") } - var promptBytes int - for _, m := range sub.Messages { - if !allowedRoles[m.Role] { + n := len(s.Task) + for _, m := range s.Messages { + if !allowedRoles[m.Role] || !utf8.ValidString(m.Content) { return errf("unknown") } - promptBytes += len(m.Content) + n += len(m.Content) } - if promptBytes > maxPromptBytes { + if n > maxPromptBytes { return errf("unknown") } - if sub.SessionID != "" { - if err := validateID(sub.SessionID); err != nil { - return err - } - } return nil } - -func validateProgressState(state string) error { - switch state { - case StateRunning, StateWaitingUser: +func validateProgressState(s string) error { + switch s { + case StateClaimed, StateRunning, StateWaitingUser: return nil - default: - return errf("unknown") } + return errf("unknown") } - -func validateSummary(summary string) error { - if len(summary) > maxSummaryBytes { - return errf("unknown") - } - if !utf8.ValidString(summary) { +func validateSummary(s string) error { + if len(s) > maxSummaryBytes || !utf8.ValidString(s) { return errf("unknown") } return nil } - -func validateResult(res Result) error { - if len(res.Text) > maxResultBytes { +func validateResult(r Result) error { + if !utf8.ValidString(r.Text) || len(r.Artifacts) > maxArtifacts || r.ErrorCode != "" && validateID(r.ErrorCode) != nil || r.Cancelled && r.ErrorCode != "" { return errf("unknown") } - if !utf8.ValidString(res.Text) { - return errf("unknown") + n := len(r.Text) + len(r.ErrorCode) + for _, a := range r.Artifacts { + if validateArtifact(a) != nil { + return errf("unknown") + } + n += len(a.URL) + len(a.Label) } - if len(res.Artifacts) > maxArtifacts { + if n > maxResultBytes { return errf("unknown") } - for _, a := range res.Artifacts { - if err := validateArtifact(a); err != nil { - return err - } - } return nil } - func validateArtifact(a Artifact) error { - if len(a.Label) > maxLabelBytes || !utf8.ValidString(a.Label) { + if len(a.Label) > maxLabelBytes || !utf8.ValidString(a.Label) || len(a.URL) > maxURLBytes || !utf8.ValidString(a.URL) || strings.Contains(a.URL, "\\") { return errf("unknown") } - u, err := url.Parse(a.URL) - if err != nil || u.Scheme != "https" || u.Host == "" { + u, e := url.Parse(a.URL) + if e != nil || u.Scheme != "https" || u.Host == "" || u.User != nil || u.Opaque != "" { return errf("unknown") } - if strings.Contains(a.URL, "\\") || strings.HasPrefix(a.URL, "file:") { - return errf("unknown") + if p := u.Port(); p != "" { + n, e := strconv.Atoi(p) + if e != nil || n < 1 || n > 65535 { + return errf("unknown") + } + } + h := strings.ToLower(strings.TrimSuffix(u.Hostname(), ".")) + if ip, e := netip.ParseAddr(h); e == nil { + ip = ip.Unmap() + if !ip.IsGlobalUnicast() || ip.IsPrivate() || ip.IsLoopback() || ip.IsLinkLocalUnicast() || ip.IsUnspecified() || ip.Zone() != "" || netip.MustParsePrefix("100.64.0.0/10").Contains(ip) { + return errf("unknown") + } + } else { + // Reject browser-style abbreviated/numeric IPv4 aliases as hostnames. + numeric := true + for _, c := range h { + if c != '.' && (c < '0' || c > '9') { + numeric = false + } + } + if numeric { + return errf("unknown") + } + if !hostPattern.MatchString(h) || !strings.Contains(h, ".") || strings.HasSuffix(h, ".localhost") || h == "localhost" || strings.HasSuffix(h, ".local") || strings.Contains(h, "..") { + return errf("unknown") + } + for _, label := range strings.Split(h, ".") { + if len(label) > 63 || strings.HasPrefix(label, "-") || strings.HasSuffix(label, "-") { + return errf("unknown") + } + } } return nil } From 9555035166a25b27ffcf906b712281b02e2dac13 Mon Sep 17 00:00:00 2001 From: Pyuyi <204892921+VeraPyuyi@users.noreply.github.com> Date: Mon, 19 Oct 2026 11:27:40 +0800 Subject: [PATCH 05/47] fix(dotqueue): reject browser numeric artifact host aliases Round2 residual I4: distinguish WHATWG numeric final labels from DNS names without integer overflow or DNS lookups. Reject hex/octal/shorthand/integer aliases with optional trailing dot, IPv4-mapped IPv6 and scoped addresses; retain canonical public IPs and ordinary DNS hosts. Add literal rejected/allowed URL table before changing validation. Behavioral RED: isolated run_checks.py dotqueue-round2-red go test ./internal/dotqueue -run ^TestArtifactHostNumericAliases$ -count=1 -v; exit 1, compiled assertions reproduce hex loopback/private bypasses. GREEN: dotqueue-round2-green go test ./internal/dotqueue -count=1 -v exit 0 (3.43s). RACE: runner --race dotqueue-round2-race go test ./internal/dotqueue -race -count=1 -v exit 0 (13.31s). Logs and JSON retained in magpie-context-verification. No full-suite rerun or DNS safety claim; I4 pending independent review. --- docs/DOT_TASK_QUEUE.md | 2 +- internal/dotqueue/artifact_host_test.go | 53 +++++++++++++++++++++++++ internal/dotqueue/validate.go | 41 ++++++++++++++----- 3 files changed, 86 insertions(+), 10 deletions(-) create mode 100644 internal/dotqueue/artifact_host_test.go diff --git a/docs/DOT_TASK_QUEUE.md b/docs/DOT_TASK_QUEUE.md index bf2ad9f..3090d47 100644 --- a/docs/DOT_TASK_QUEUE.md +++ b/docs/DOT_TASK_QUEUE.md @@ -14,7 +14,7 @@ Progress accepts claimed, running or waiting_user, stores `Job.Summary` durably Recover is an explicit owner-startup action, never a routine client Open action. It changes claimed/running/waiting_user/cancel_requested to outcome_unknown once, preserving token and notice identity. Unknown work cannot be claimed or progressed into an active state. The original token can reconcile via Complete, including explicit cancellation acknowledgment. Final completed/failed/cancelled results permit identical retries and reject changed results. No lease, automatic reassignment or automatic execution retry exists. -Result content has a total 262144-byte budget across Text, ErrorCode, artifact URLs and labels. ErrorCode, when present, uses the ID syntax (maximum 128 bytes), representing a stable caller-defined error identifier rather than diagnostic prose. Cancelled and ErrorCode cannot both be set. At most 16 artifacts are allowed, each label at most 256 bytes and URL at most 2048 bytes. All strings must be valid UTF-8. Artifacts require HTTPS without userinfo; localhost/local names, single-label names, private/loopback/link-local/unspecified/non-unicast literal addresses are rejected. Hostnames are not DNS-resolved here. Any future fetcher must validate DNS addresses at connection time and constrain redirects; this backend's syntactic policy is not a network safety guarantee. +Result content has a total 262144-byte budget across Text, ErrorCode, artifact URLs and labels. ErrorCode, when present, uses the ID syntax (maximum 128 bytes), representing a stable caller-defined error identifier rather than diagnostic prose. Cancelled and ErrorCode cannot both be set. At most 16 artifacts are allowed, each label at most 256 bytes and URL at most 2048 bytes. All strings must be valid UTF-8. Artifacts require HTTPS without userinfo; localhost/local names, single-label names, private/loopback/link-local/unspecified/non-unicast literal addresses are rejected. Noncanonical hosts whose final label is decimal or hexadecimal numeric syntax (including octal-looking labels, integer/shorthand forms and an optional trailing dot) are rejected rather than interpreted as DNS names; canonical public IPv4 remains allowed. IPv4-mapped IPv6 and scoped/zone-ID addresses are rejected outright. Ordinary DNS names, including numeric-looking non-final labels, remain allowed. Hostnames are not DNS-resolved here. Any future fetcher must validate DNS addresses at connection time and constrain redirects; this backend's syntactic policy is not a network safety guarantee. State is AES-256-GCM encrypted with a random nonce. The `D`/version header is authenticated as additional data; encrypted JSON also requires state_version 1. Nested schemas reject unknown, duplicate, missing required, null and wrongly typed fields. Every loaded job validates payload/result budgets and digests, lifecycle/claim/cancellation/result consistency, timestamps, unique events/tokens, capacity, order and the job/request-index bijection. Empty arrays are persisted explicitly. Earlier development fixtures without Summary or header AAD are incompatible and fail closed; there is no silent migration or reset. diff --git a/internal/dotqueue/artifact_host_test.go b/internal/dotqueue/artifact_host_test.go new file mode 100644 index 0000000..a52b43e --- /dev/null +++ b/internal/dotqueue/artifact_host_test.go @@ -0,0 +1,53 @@ +package dotqueue + +import "testing" + +func TestArtifactHostNumericAliases(t *testing.T) { + cases := []struct { + url string + allowed bool + }{ + {"https://0x7f.0.0.1/a", false}, + {"https://0xa.0.0.1/a", false}, + {"https://0X7F.0X0.0.1/a", false}, + {"https://0x7f.1/a", false}, + {"https://0x7f000001/a", false}, + {"https://0x7f000001./a", false}, + {"https://0x7f.0.0.1./a", false}, + {"https://0177.0.0.1/a", false}, + {"https://0177.1./a", false}, + {"https://127.1/a", false}, + {"https://127.0.1./a", false}, + {"https://2130706433/a", false}, + {"https://2130706433./a", false}, + {"https://0x08080808/a", false}, + {"https://010.010.010.010/a", false}, + {"https://example.123/a", false}, + {"https://example.0x7f/a", false}, + {"https://example.0x/a", false}, + {"https://[::ffff:127.0.0.1]/a", false}, + {"https://[::ffff:7f00:1]/a", false}, + {"https://[::ffff:8.8.8.8]/a", false}, + {"https://[::ffff:808:808]/a", false}, + {"https://[fe80::1%25eth0]/a", false}, + {"https://[2606:4700:4700::1111%25eth0]/a", false}, + {"https://example.com/a", true}, + {"https://Example.COM./a", true}, + {"https://cdn123.example.com/a", true}, + {"https://0x7f.example.com/a", true}, + {"https://123.example.com/a", true}, + {"https://example.0xg/a", true}, + {"https://example.123abc/a", true}, + {"https://8.8.8.8/a", true}, + {"https://8.8.8.8./a", true}, + {"https://[2606:4700:4700::1111]/a", true}, + } + for _, tc := range cases { + t.Run(tc.url, func(t *testing.T) { + err := validateArtifact(Artifact{URL: tc.url, Label: "artifact"}) + if (err == nil) != tc.allowed { + t.Fatalf("allowed=%v, got error %v", tc.allowed, err) + } + }) + } +} diff --git a/internal/dotqueue/validate.go b/internal/dotqueue/validate.go index 0c366a6..eb2ca79 100644 --- a/internal/dotqueue/validate.go +++ b/internal/dotqueue/validate.go @@ -87,6 +87,29 @@ func validateResult(r Result) error { } return nil } + +// endsInIPv4Number implements the numeric final-label distinction used by +// WHATWG hosts without parsing into a bounded integer (overflow must not turn +// an alias into a DNS name). The caller already removed one trailing dot and +// lowercased the host. All-decimal labels include octal-looking/invalid octal +// forms; a 0x prefix with only hex digits includes the empty-digit zero form. +func endsInIPv4Number(host string) bool { + last := host[strings.LastIndexByte(host, '.')+1:] + if last == "" { + return false + } + digits, alphabet := last, "0123456789" + if strings.HasPrefix(last, "0x") { + digits, alphabet = last[2:], "0123456789abcdef" + } + for _, c := range digits { + if !strings.ContainsRune(alphabet, c) { + return false + } + } + return true +} + func validateArtifact(a Artifact) error { if len(a.Label) > maxLabelBytes || !utf8.ValidString(a.Label) || len(a.URL) > maxURLBytes || !utf8.ValidString(a.URL) || strings.Contains(a.URL, "\\") { return errf("unknown") @@ -103,19 +126,19 @@ func validateArtifact(a Artifact) error { } h := strings.ToLower(strings.TrimSuffix(u.Hostname(), ".")) if ip, e := netip.ParseAddr(h); e == nil { - ip = ip.Unmap() + // Reject mapped and scoped forms outright rather than letting consumers + // interpret an IPv4 destination through an IPv6 spelling. + if ip.Is4In6() || ip.Zone() != "" { + return errf("unknown") + } if !ip.IsGlobalUnicast() || ip.IsPrivate() || ip.IsLoopback() || ip.IsLinkLocalUnicast() || ip.IsUnspecified() || ip.Zone() != "" || netip.MustParsePrefix("100.64.0.0/10").Contains(ip) { return errf("unknown") } } else { - // Reject browser-style abbreviated/numeric IPv4 aliases as hostnames. - numeric := true - for _, c := range h { - if c != '.' && (c < '0' || c > '9') { - numeric = false - } - } - if numeric { + // WHATWG treats a domain ending in a number as IPv4 input, including + // hex, octal, shortened and integer spellings. Only canonical IPs may + // reach the address-policy branch above; never accept aliases as DNS. + if endsInIPv4Number(h) { return errf("unknown") } if !hostPattern.MatchString(h) || !strings.Contains(h, ".") || strings.HasSuffix(h, ".localhost") || h == "localhost" || strings.HasSuffix(h, ".local") || strings.Contains(h, "..") { From 1893738a2ca5c8d1ff81c6f4efe5686b80a6cddb Mon Sep 17 00:00:00 2001 From: Pyuyi <204892921+VeraPyuyi@users.noreply.github.com> Date: Mon, 19 Oct 2026 11:30:24 +0800 Subject: [PATCH 06/47] feat(dotevents): persist secure finite webhook subscriptions Implement independent encrypted subscription state, generation-checked JSON callback verification, official event occurrence fields, Standard Webhooks exact-byte signing and secret rotation. Pin each connection to validated public DNS addresses with normal TLS verification and no proxies or redirects. Reserve per-subscription delivery attempts atomically and persist bounded retry and acknowledgement state. Evidence: behavior-first scaffold test failed on invalid subscriptions and reserved IP acceptance; official-wire regression failed on discovery and JSON challenge verification; reserved-address and quoted-TTL regressions failed before fixes. Final go test ./internal/dotevents -count=1 passed (1.811s); go test -race ./internal/dotevents -count=1 passed (4.197s, CGO/MinGW); go vet ./internal/dotevents and git diff --cached --check passed. All callbacks use local TLS fixtures; production dial guard tested independently with offline resolver/dial functions. Full suite deferred to integration controller. --- docs/DOT_EVENTS.md | 23 +++ internal/dotevents/crypto.go | 62 +++++++ internal/dotevents/delivery.go | 203 ++++++++++++++++++++ internal/dotevents/errors.go | 41 +++++ internal/dotevents/events_test.go | 251 +++++++++++++++++++++++++ internal/dotevents/files_unix.go | 42 +++++ internal/dotevents/files_windows.go | 65 +++++++ internal/dotevents/identity.go | 13 ++ internal/dotevents/lifecycle_test.go | 236 ++++++++++++++++++++++++ internal/dotevents/manager.go | 265 +++++++++++++++++++++++++++ internal/dotevents/multiple_test.go | 85 +++++++++ internal/dotevents/params.go | 112 +++++++++++ internal/dotevents/security_test.go | 68 +++++++ internal/dotevents/state.go | 170 +++++++++++++++++ internal/dotevents/store.go | 229 +++++++++++++++++++++++ internal/dotevents/store_test.go | 62 +++++++ internal/dotevents/testutil_test.go | 56 ++++++ internal/dotevents/types.go | 53 ++++++ internal/dotevents/validate.go | 213 +++++++++++++++++++++ internal/dotevents/wire_test.go | 47 +++++ 20 files changed, 2296 insertions(+) create mode 100644 docs/DOT_EVENTS.md create mode 100644 internal/dotevents/crypto.go create mode 100644 internal/dotevents/delivery.go create mode 100644 internal/dotevents/errors.go create mode 100644 internal/dotevents/events_test.go create mode 100644 internal/dotevents/files_unix.go create mode 100644 internal/dotevents/files_windows.go create mode 100644 internal/dotevents/identity.go create mode 100644 internal/dotevents/lifecycle_test.go create mode 100644 internal/dotevents/manager.go create mode 100644 internal/dotevents/multiple_test.go create mode 100644 internal/dotevents/params.go create mode 100644 internal/dotevents/security_test.go create mode 100644 internal/dotevents/state.go create mode 100644 internal/dotevents/store.go create mode 100644 internal/dotevents/store_test.go create mode 100644 internal/dotevents/testutil_test.go create mode 100644 internal/dotevents/types.go create mode 100644 internal/dotevents/validate.go create mode 100644 internal/dotevents/wire_test.go diff --git a/docs/DOT_EVENTS.md b/docs/DOT_EVENTS.md new file mode 100644 index 0000000..da7c9e4 --- /dev/null +++ b/docs/DOT_EVENTS.md @@ -0,0 +1,23 @@ +# Dot task event subscriptions + +`internal/dotevents` implements the subscription store and outbound webhook transport for MCP Events (2026-07-28), following the saved routing design and OpenAI's [MCP Events guide](https://developers.openai.com/plugins/build/mcp-events). Integration into MCP discovery and the runtime worker is a separate task. This package does not implement poll, stream, gap recovery, queue execution, or model readiness. + +`Open(absoluteDir, key, delivery)` requires a caller-supplied 32-byte key. A nil delivery or zero-value `Delivery` uses public HTTPS with normal certificate verification. The package has no production switch to permit private callbacks. Tests inject an unexported HTTP client with explicit local fixture routing and trusted fixture certificates. + +The authenticated adapter supplies `principal`; the caller's `arguments.queue_id` must equal that principal. `List` advertises `dot.task.available` before a subscription exists. `Params` accepts `name`, `arguments:{queue_id}`, `delivery:{mode:"webhook",url,secret}`, optional null `cursor`, and raw `ttlMs`. Its JSON decoder rejects unknown/duplicate fields and null required objects. The adapter must enforce the request byte limit before decoding too. Omitted or null TTL means one hour, with an allowed range of one minute to one day. No non-expiring grant is supported. Secrets use `whsec_` plus standard base64 encoding of 24–64 bytes. URLs are limited to 2048 bytes and params to 16 KiB. + +Subscription identity is SHA-256 over the principal, event, arguments, mode, and exact callback URL, excluding the secret and TTL. Refresh keeps that identity. The encrypted state is persisted before callback verification. Every refresh gets a random generation; publishing the verification result requires the same generation after network I/O. Pause invalidates all callback verifications and generations, including across disable/re-enable. Unsubscribe selects the same identity without requiring a secret and is idempotent. The integration adapter must map the official unsubscribe request into this internal selection struct. + +Verification posts signed `{ "type":"verification", "challenge":"" }`. The unique nonce is used for one attempt bounded to ten seconds, below the five-minute verification maximum. A successful response must be 2xx with bounded JSON `{ "challenge":"" }`; comparison uses constant time. An event occurrence contains `eventId`, `name`, an RFC3339 timestamp, `data:{queue_id,job_id,mode}`, and `cursor:null`. It never includes task prompts. Occurrence bytes are persisted on the first attempt and reused for retries. + +Standard Webhooks signs the exact UTF-8 bytes with HMAC-SHA256 over `webhook-id.webhook-timestamp.body`. `webhook-timestamp` uses Unix seconds. The `webhook-id` is derived from subscription ID plus notice event ID, so each subscription has a distinct stable delivery identity. Each POST includes `X-MCP-Subscription-Id`. Refreshing a secret preserves the previous secret for dual signatures during a 60-second grace window. Expired subscriptions do not deliver. + +Every `Deliver` visits all matching verified subscriptions in sorted identity order, making at most one HTTP attempt per subscription. It reserves attempts atomically under the cross-process state lock before I/O. Parallel workers receive `in_flight` with a retry delay while a reservation is active; a crashed worker consumes an attempt and its reservation expires after eleven seconds. A 2xx records receipt only. 410/413 and other 4xx are terminal, except 429. Network failures, 5xx, and 429 permit three retries after 1s, 2s, and 4s, for four total attempts. No sleeps or background worker exist here: the caller/outbox schedules `Error.RetryAfter`. An early call returns the remaining delay without sending. Terminal and acknowledged records suppress further sends. There is no exactly-once guarantee after an ambiguous callback outcome. Storage capacity is 100 subscriptions and 1000 delivery records; reaching the delivery bound fails closed and requires a future explicit retention policy. + +Outbound transport disables environment proxies, redirects, connection reuse, and implicit DNS during socket dialing. Every new connection resolves with context, rejects the entire answer set if any address is non-public, then dials validated literal IPs while HTTP retains the original hostname for TLS SNI and certificate checks. Loopback, private, link-local, CGNAT, documentation, reserved, multicast, unspecified, IPv6 transition, and special-purpose ranges are rejected. Query strings are allowed and never copied into errors. Timeouts are at most ten seconds; response headers and verification bodies are limited to 16 KiB; event response bodies are discarded without reading; event bodies are capped at 256 KiB. + +State uses AES-256-GCM, private directory/file permissions (protected current-user DACL on Windows), regular-file and hardlink checks, a five-second bounded flock, synced temporary files, and atomic replacement. Separate files are `events.state.de`, `dot_events.lock`, and `dot_events.initialized`. Missing state after initialization and invalid ciphertext fail closed. No queue package private functions or files are used. + +Errors are stable `dot_events_` values with optional retry duration and no callback URL, secret, response page, or filesystem path. `Status` distinguishes subscription and callback verification; `DotReady` is always false. The runtime must establish real readiness separately through authenticated claim/completion of a probe, and propagate pairing authorization changes into `SetEnabled` before allowing operations. + +Offline verification: `go test ./internal/dotevents` and `go test -race ./internal/dotevents`. Tests cover official wire fields, JSON challenge replies, exact HMAC bytes, rotation, persistence, expiry, unsubscribe, pause during verification/delivery, competing refresh, duplicate acknowledgements, multiple subscriptions, atomic reservations, bounded retries, production public-address checks, DNS rebinding, redirect rejection, cross-process locks, and corrupt/missing state. Full repository tests belong to the integration controller. diff --git a/internal/dotevents/crypto.go b/internal/dotevents/crypto.go new file mode 100644 index 0000000..1ab1028 --- /dev/null +++ b/internal/dotevents/crypto.go @@ -0,0 +1,62 @@ +package dotevents + +import ( + "crypto/aes" + "crypto/cipher" + "crypto/rand" + "io" +) + +const envelopeVersion = 1 + +func seal(key []byte, plaintext []byte) ([]byte, error) { + if len(key) != 32 { + return nil, errf("unknown") + } + block, err := aes.NewCipher(key) + if err != nil { + return nil, errf("unknown") + } + gcm, err := cipher.NewGCM(block) + if err != nil { + return nil, errf("unknown") + } + nonce := make([]byte, gcm.NonceSize()) + if _, err := io.ReadFull(rand.Reader, nonce); err != nil { + return nil, errf("unknown") + } + ciphertext := gcm.Seal(nil, nonce, plaintext, nil) + out := make([]byte, 2+len(nonce)+len(ciphertext)) + out[0] = 'E' + out[1] = envelopeVersion + copy(out[2:], nonce) + copy(out[2+len(nonce):], ciphertext) + return out, nil +} + +func openEnvelope(key []byte, data []byte) ([]byte, error) { + if len(key) != 32 { + return nil, errf("corrupt") + } + if len(data) < 3 || data[0] != 'E' || data[1] != envelopeVersion { + return nil, errf("corrupt") + } + block, err := aes.NewCipher(key) + if err != nil { + return nil, errf("corrupt") + } + gcm, err := cipher.NewGCM(block) + if err != nil { + return nil, errf("corrupt") + } + if len(data) < 2+gcm.NonceSize() { + return nil, errf("corrupt") + } + nonce := data[2 : 2+gcm.NonceSize()] + ciphertext := data[2+gcm.NonceSize():] + plain, err := gcm.Open(nil, nonce, ciphertext, nil) + if err != nil { + return nil, errf("corrupt") + } + return plain, nil +} diff --git a/internal/dotevents/delivery.go b/internal/dotevents/delivery.go new file mode 100644 index 0000000..39d5c00 --- /dev/null +++ b/internal/dotevents/delivery.go @@ -0,0 +1,203 @@ +package dotevents + +import ( + "bytes" + "context" + "crypto/hmac" + "crypto/rand" + "crypto/sha256" + "crypto/subtle" + "encoding/base64" + "encoding/hex" + "encoding/json" + "io" + "net" + "net/http" + "net/netip" + "strconv" + "strings" + "time" +) + +const ( + maxHTTPTimeout = 10 * time.Second + maxResponseBytes = 16 * 1024 + maxEventBodyBytes = 256 * 1024 + secretRotateGrace = 60 * time.Second +) + +var retrySchedule = []time.Duration{time.Second, 2 * time.Second, 4 * time.Second} + +// Delivery's zero value uses the public HTTPS transport. Custom transports are +// deliberately private; offline TLS fixtures construct them inside package tests. +type Delivery struct{ client *http.Client } + +func (d *Delivery) httpClient() *http.Client { + if d != nil && d.client != nil { + return d.client + } + return &http.Client{Timeout: maxHTTPTimeout, Transport: publicTransport(net.DefaultResolver.LookupIPAddr, (&net.Dialer{Timeout: maxHTTPTimeout}).DialContext), CheckRedirect: noRedirect} +} +func noRedirect(*http.Request, []*http.Request) error { return http.ErrUseLastResponse } + +type resolveFunc func(context.Context, string) ([]net.IPAddr, error) +type dialFunc func(context.Context, string, string) (net.Conn, error) + +func publicTransport(resolve resolveFunc, dial dialFunc) *http.Transport { + return &http.Transport{Proxy: nil, DisableKeepAlives: true, TLSHandshakeTimeout: maxHTTPTimeout, ResponseHeaderTimeout: maxHTTPTimeout, MaxResponseHeaderBytes: maxResponseBytes, DialContext: func(ctx context.Context, network, address string) (net.Conn, error) { + host, port, e := net.SplitHostPort(address) + if e != nil { + return nil, errf("callback") + } + var addrs []net.IPAddr + if ip, e := netip.ParseAddr(host); e == nil { + addrs = []net.IPAddr{{IP: ip.AsSlice(), Zone: ip.Zone()}} + } else { + addrs, e = resolve(ctx, host) + if e != nil { + return nil, errf("callback") + } + } + if !publicAddrs(addrs) { + return nil, errf("callback") + } + // Every answer must pass validation before any connection is attempted. Dial + // literal addresses so DNS cannot be consulted again by the socket layer. + for _, a := range addrs { + conn, e := dial(ctx, network, net.JoinHostPort(a.IP.String(), port)) + if e == nil { + return conn, nil + } + if ctx.Err() != nil { + break + } + } + return nil, errf("network") + }} +} +func (d *Delivery) post(ctx context.Context, sub *subscriptionRecord, id string, body []byte) (*http.Response, error) { + if len(body) > maxEventBodyBytes { + return nil, errf("unknown") + } + if e := validateCallbackURL(sub.Params.Delivery.URL); e != nil { + return nil, e + } + ctx, cancel := context.WithTimeout(ctx, maxHTTPTimeout) + req, e := http.NewRequestWithContext(ctx, http.MethodPost, sub.Params.Delivery.URL, bytes.NewReader(body)) + if e != nil { + cancel() + return nil, errf("callback") + } + stamp := time.Now().UTC().Unix() + setWebhookHeaders(req, id, stamp, signPayload(sub, id, body, stamp), sub.ID) + resp, e := d.httpClient().Do(req) + if e != nil { + cancel() + return nil, errf("network") + } + resp.Body = &cancelBody{ReadCloser: resp.Body, cancel: cancel} + return resp, nil +} + +type cancelBody struct { + io.ReadCloser + cancel context.CancelFunc +} + +func (b *cancelBody) Close() error { defer b.cancel(); return b.ReadCloser.Close() } +func (d *Delivery) verify(ctx context.Context, sub *subscriptionRecord) error { + challenge, e := randomToken(32) + if e != nil { + return e + } + id, e := randomToken(16) + if e != nil { + return e + } + resp, e := d.post(ctx, sub, id, mustMarshal(map[string]string{"type": "verification", "challenge": challenge})) + if e != nil { + return e + } + defer resp.Body.Close() + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return errf("verification") + } + data, e := io.ReadAll(io.LimitReader(resp.Body, maxResponseBytes+1)) + var reply struct { + Challenge string `json:"challenge"` + } + if e != nil || len(data) > maxResponseBytes || json.Unmarshal(data, &reply) != nil || subtle.ConstantTimeCompare([]byte(reply.Challenge), []byte(challenge)) != 1 { + return errf("verification") + } + return nil +} +func (d *Delivery) deliverBytes(ctx context.Context, sub *subscriptionRecord, id string, body []byte) (string, error) { + resp, e := d.post(ctx, sub, deliveryID(sub.ID, id), body) + if e != nil { + return "network", errf("network") + } + defer resp.Body.Close() + // Event response bodies have no protocol meaning. Do not read arbitrary error + // pages; closing the non-reused connection bounds both memory and work. + code := strconv.Itoa(resp.StatusCode) + switch { + case resp.StatusCode >= 200 && resp.StatusCode < 300: + return code, nil + case resp.StatusCode == 410 || resp.StatusCode == 413: + return code, errf("terminal") + case resp.StatusCode == 429: + return code, errf("rate_limited") + case resp.StatusCode >= 500: + return code, errf("server") + default: + return code, errf("client") + } +} +func eventBodyBytes(n Notice) []byte { + return mustMarshal(struct { + EventID string `json:"eventId"` + Name string `json:"name"` + Timestamp time.Time `json:"timestamp"` + Data struct { + QueueID string `json:"queue_id"` + JobID string `json:"job_id"` + Mode string `json:"mode"` + } `json:"data"` + Cursor any `json:"cursor"` + }{EventID: n.EventID, Name: EventName, Timestamp: time.Now().UTC(), Data: struct { + QueueID string `json:"queue_id"` + JobID string `json:"job_id"` + Mode string `json:"mode"` + }{n.QueueID, n.JobID, n.Mode}}) +} +func setWebhookHeaders(req *http.Request, id string, stamp int64, sigs []string, subID string) { + req.Header.Set("Content-Type", "application/json") + req.Header.Set("webhook-id", id) + req.Header.Set("webhook-timestamp", strconv.FormatInt(stamp, 10)) + req.Header.Set("webhook-signature", strings.Join(sigs, " ")) + req.Header.Set("X-MCP-Subscription-Id", subID) +} +func signPayload(sub *subscriptionRecord, id string, body []byte, stamp int64) []string { + msg := id + "." + strconv.FormatInt(stamp, 10) + "." + string(body) + out := []string{signature(sub.Secret, msg)} + if sub.PreviousSecret != "" && time.Now().Before(sub.RotateUntil) { + out = append(out, signature(sub.PreviousSecret, msg)) + } + return out +} +func signature(secret, msg string) string { + key, e := decodeSecret(secret) + if e != nil { + return "" + } + mac := hmac.New(sha256.New, key) + mac.Write([]byte(msg)) + return "v1," + base64.StdEncoding.EncodeToString(mac.Sum(nil)) +} +func randomToken(n int) (string, error) { + b := make([]byte, n) + if _, e := rand.Read(b); e != nil { + return "", errf("unknown") + } + return hex.EncodeToString(b), nil +} diff --git a/internal/dotevents/errors.go b/internal/dotevents/errors.go new file mode 100644 index 0000000..e6b3dae --- /dev/null +++ b/internal/dotevents/errors.go @@ -0,0 +1,41 @@ +package dotevents + +import "time" + +type Error struct { + Code string + RetryAfter time.Duration +} + +func (e *Error) Error() string { + if e == nil || e.Code == "" { + return "dot_events_unknown" + } + return "dot_events_" + e.Code +} + +func errf(code string) error { return &Error{Code: code} } + +func retryErr(code string, after time.Duration) error { + return &Error{Code: code, RetryAfter: after} +} + +func asEventsError(err error) *Error { + if err == nil { + return nil + } + if ee, ok := err.(*Error); ok { + return ee + } + return &Error{Code: "unknown"} +} + +func wrapUnknown(err error) error { + if err == nil { + return nil + } + if _, ok := err.(*Error); ok { + return err + } + return errf("unknown") +} diff --git a/internal/dotevents/events_test.go b/internal/dotevents/events_test.go new file mode 100644 index 0000000..d011dce --- /dev/null +++ b/internal/dotevents/events_test.go @@ -0,0 +1,251 @@ +package dotevents + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + "io" + "net/http" + "net/netip" + "path/filepath" + "strings" + "sync" + "testing" +) + +func TestInvalidSubscriptions(t *testing.T) { + tests := []struct { + name string + principal string + mutate func(*Params) + }{ + {"principal", "bob", func(*Params) {}}, + {"event", "alice", func(p *Params) { p.Name = "other" }}, + {"secret", "alice", func(p *Params) { p.Delivery.Secret = "whsec_bad" }}, + {"ttl", "alice", func(p *Params) { p.TTL = json.RawMessage(`1`) }}, + {"ttl_fraction", "alice", func(p *Params) { p.TTL = json.RawMessage(`60000.5`) }}, + {"cursor", "alice", func(p *Params) { p.Cursor = json.RawMessage(`"resume"`) }}, + {"url", "alice", func(p *Params) { p.Delivery.URL = "http://127.0.0.1" }}, + {"fragment", "alice", func(p *Params) { p.Delivery.URL = "https://callback.example/#x" }}, + {"userinfo", "alice", func(p *Params) { p.Delivery.URL = "https://user@callback.example/" }}, + {"localhost", "alice", func(p *Params) { p.Delivery.URL = "https://localhost/" }}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mgr, _ := openTestManager(t) + p := testParams(tt.principal) + tt.mutate(&p) + if _, err := mgr.Subscribe(context.Background(), tt.principal, p); err == nil { + t.Fatal("accepted invalid subscription") + } + }) + } +} + +func TestPublicGuard(t *testing.T) { + for _, ip := range []string{"127.0.0.1", "10.1.1.1", "169.254.169.254", "100.64.1.1", "192.0.2.1", "198.51.100.1", "203.0.113.1", "224.0.0.0", "0.0.0.0", "::1", "fc00::1", "fe80::1", "2001:db8::1", "::ffff:127.0.0.1", "198.18.0.1", "240.0.0.1"} { + addr, err := netip.ParseAddr(ip) + if err != nil { + t.Fatal(err) + } + if publicIP(addr) { + t.Errorf("accepted %s", ip) + } + } +} + +func TestSubscribeVerifyDeliverAndPersist(t *testing.T) { + var verifyChallenge string + var eventBody []byte + var signatures []string + mgr, dir, close, _ := withFixtureServerURL(t, func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(io.LimitReader(r.Body, maxResponseBytes+1)) + if strings.Contains(string(body), "verification") { + var payload map[string]string + _ = json.Unmarshal(body, &payload) + verifyChallenge = payload["challenge"] + w.WriteHeader(http.StatusOK) + _ = json.NewEncoder(w).Encode(map[string]string{"challenge": verifyChallenge}) + return + } + eventBody = append([]byte(nil), body...) + signatures = strings.Fields(r.Header.Get("webhook-signature")) + w.WriteHeader(http.StatusOK) + }) + defer close() + sub, err := mgr.Subscribe(context.Background(), "alice", testParams("alice")) + if err != nil || !sub.CallbackVerified || verifyChallenge == "" { + t.Fatalf("subscribe verify failed: %v %#v", err, sub) + } + status, err := mgr.Status("alice", "alice") + if err != nil || !status.Subscribed || !status.CallbackVerified || status.DotReady { + t.Fatalf("unexpected status %#v", status) + } + notice := Notice{EventID: "evt.1", QueueID: "alice", JobID: "job.1", Mode: "answer"} + if err := mgr.Deliver(context.Background(), "alice", "alice", notice); err != nil { + t.Fatal(err) + } + var occurrence map[string]json.RawMessage + if json.Unmarshal(eventBody, &occurrence) != nil || string(occurrence["eventId"]) != `"evt.1"` { + t.Fatalf("event body mismatch: %q", eventBody) + } + if len(signatures) == 0 { + t.Fatal("missing signature") + } + reopened, err := Open(dir, testKey, nil) + if err != nil { + t.Fatal(err) + } + again, err := reopened.Status("alice", "alice") + if err != nil || !again.Acknowledged { + t.Fatalf("persisted ack missing: %#v", again) + } +} + +func TestChallengeMismatchAndOverread(t *testing.T) { + mgr, _, close, _ := withFixtureServerURL(t, func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(io.LimitReader(r.Body, maxResponseBytes+1)) + if strings.Contains(string(body), "verification") { + w.WriteHeader(http.StatusOK) + _, _ = w.Write(bytes.Repeat([]byte("x"), maxResponseBytes+1)) + return + } + w.WriteHeader(http.StatusOK) + }) + defer close() + if _, err := mgr.Subscribe(context.Background(), "alice", testParams("alice")); err == nil { + t.Fatal("accepted overread challenge") + } +} + +func TestDuplicateEventAck(t *testing.T) { + mgr, _, close, _ := withFixtureServerURL(t, func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(io.LimitReader(r.Body, maxResponseBytes+1)) + if strings.Contains(string(body), "verification") { + var payload map[string]string + _ = json.Unmarshal(body, &payload) + _ = json.NewEncoder(w).Encode(map[string]string{"challenge": payload["challenge"]}) + return + } + w.WriteHeader(http.StatusOK) + }) + defer close() + if _, err := mgr.Subscribe(context.Background(), "alice", testParams("alice")); err != nil { + t.Fatal(err) + } + notice := Notice{EventID: "evt.dup", QueueID: "alice", JobID: "job.1", Mode: "task"} + if err := mgr.Deliver(context.Background(), "alice", "alice", notice); err != nil { + t.Fatal(err) + } + if err := mgr.Deliver(context.Background(), "alice", "alice", notice); err != nil { + t.Fatal(err) + } + status, _ := mgr.Status("alice", "alice") + if status.Attempts != 1 { + t.Fatalf("duplicate ack should not increment attempts: %#v", status) + } +} + +func TestSecretRotationGrace(t *testing.T) { + var seen [][]string + mgr, _, close, _ := withFixtureServerURL(t, func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(io.LimitReader(r.Body, maxResponseBytes+1)) + if strings.Contains(string(body), "verification") { + var payload map[string]string + _ = json.Unmarshal(body, &payload) + _ = json.NewEncoder(w).Encode(map[string]string{"challenge": payload["challenge"]}) + return + } + seen = append(seen, strings.Fields(r.Header.Get("webhook-signature"))) + w.WriteHeader(http.StatusOK) + }) + defer close() + first := testParams("alice") + if _, err := mgr.Subscribe(context.Background(), "alice", first); err != nil { + t.Fatal(err) + } + second := first + second.Delivery.Secret = "whsec_" + base64.StdEncoding.EncodeToString(bytes.Repeat([]byte("b"), 32)) + if _, err := mgr.Subscribe(context.Background(), "alice", second); err != nil { + t.Fatal(err) + } + notice := Notice{EventID: "evt.rotate", QueueID: "alice", JobID: "job.2", Mode: "answer"} + if err := mgr.Deliver(context.Background(), "alice", "alice", notice); err != nil { + t.Fatal(err) + } + if len(seen) == 0 || len(seen[0]) < 2 { + t.Fatalf("expected dual signatures during grace, got %#v", seen) + } +} + +func TestDisabledBlocksDelivery(t *testing.T) { + mgr, _, close, _ := withFixtureServerURL(t, func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(io.LimitReader(r.Body, maxResponseBytes+1)) + if strings.Contains(string(body), "verification") { + var payload map[string]string + _ = json.Unmarshal(body, &payload) + _ = json.NewEncoder(w).Encode(map[string]string{"challenge": payload["challenge"]}) + } + w.WriteHeader(http.StatusOK) + }) + defer close() + if _, err := mgr.Subscribe(context.Background(), "alice", testParams("alice")); err != nil { + t.Fatal(err) + } + if err := mgr.SetEnabled(false); err != nil { + t.Fatal(err) + } + if err := mgr.Deliver(context.Background(), "alice", "alice", Notice{EventID: "evt.off", QueueID: "alice", JobID: "job.1", Mode: "answer"}); asEventsError(err).Code != "disabled" { + t.Fatalf("expected disabled, got %v", err) + } +} + +func TestConcurrentSubscribeAndDeliver(t *testing.T) { + mgr, dir, close, _ := withFixtureServerURL(t, func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(io.LimitReader(r.Body, maxResponseBytes+1)) + if strings.Contains(string(body), "verification") { + var payload map[string]string + _ = json.Unmarshal(body, &payload) + _ = json.NewEncoder(w).Encode(map[string]string{"challenge": payload["challenge"]}) + return + } + w.WriteHeader(http.StatusOK) + }) + defer close() + abs, err := filepath.Abs(dir) + if err != nil { + t.Fatal(err) + } + other, err := Open(abs, testKey, mgr.delivery) + if err != nil { + t.Fatal(err) + } + var wg sync.WaitGroup + for i := 0; i < 8; i++ { + wg.Add(1) + go func() { + defer wg.Done() + _, err := mgr.Subscribe(context.Background(), "alice", testParams("alice")) + if err != nil && asEventsError(err).Code != "stale" { + t.Errorf("subscribe: %v", err) + } + }() + } + wg.Wait() + notice := Notice{EventID: "evt.race", QueueID: "alice", JobID: "job.race", Mode: "answer"} + for i := 0; i < 8; i++ { + wg.Add(1) + go func() { + defer wg.Done() + if err := other.Deliver(context.Background(), "alice", "alice", notice); err != nil && asEventsError(err).Code != "in_flight" { + t.Errorf("deliver: %v", err) + } + }() + } + wg.Wait() + status, _ := mgr.Status("alice", "alice") + if !status.Acknowledged || status.Attempts != 1 { + t.Fatalf("race status %#v", status) + } +} diff --git a/internal/dotevents/files_unix.go b/internal/dotevents/files_unix.go new file mode 100644 index 0000000..33fd2ba --- /dev/null +++ b/internal/dotevents/files_unix.go @@ -0,0 +1,42 @@ +//go:build !windows + +package dotevents + +import ( + "os" + "syscall" +) + +func privatePath(path string, directory bool) error { + mode := os.FileMode(0600) + if directory { + mode = 0700 + } + return os.Chmod(path, mode) +} + +func rejectLink(info os.FileInfo) error { + if info.Mode()&os.ModeSymlink != 0 { + return errf("unknown") + } + return nil +} + +func rejectHardlink(f *os.File, info os.FileInfo) error { + stat, ok := info.Sys().(*syscall.Stat_t) + if !ok || stat.Nlink != 1 { + return errf("unknown") + } + return nil +} + +func replaceFile(from, to string) error { return os.Rename(from, to) } + +func syncDirectory(path string) error { + f, err := os.Open(path) + if err != nil { + return err + } + defer f.Close() + return f.Sync() +} diff --git a/internal/dotevents/files_windows.go b/internal/dotevents/files_windows.go new file mode 100644 index 0000000..e2fb03b --- /dev/null +++ b/internal/dotevents/files_windows.go @@ -0,0 +1,65 @@ +//go:build windows + +package dotevents + +import ( + "os" + "syscall" + + "golang.org/x/sys/windows" +) + +func privatePath(path string, directory bool) error { + user, err := windows.GetCurrentProcessToken().GetTokenUser() + if err != nil { + return err + } + inherit := "" + if directory { + inherit = "OICI" + } + sd, err := windows.SecurityDescriptorFromString("D:P(A;" + inherit + ";FA;;;" + user.User.Sid.String() + ")") + if err != nil { + return err + } + acl, _, err := sd.DACL() + if err != nil { + return err + } + return windows.SetNamedSecurityInfo(path, windows.SE_FILE_OBJECT, windows.DACL_SECURITY_INFORMATION|windows.PROTECTED_DACL_SECURITY_INFORMATION, nil, nil, acl, nil) +} + +func rejectLink(info os.FileInfo) error { + if info.Mode()&os.ModeSymlink != 0 { + return errf("unknown") + } + if attr, ok := info.Sys().(*syscall.Win32FileAttributeData); ok && attr.FileAttributes&windows.FILE_ATTRIBUTE_REPARSE_POINT != 0 { + return errf("unknown") + } + return nil +} + +func rejectHardlink(f *os.File, _ os.FileInfo) error { + var info windows.ByHandleFileInformation + if err := windows.GetFileInformationByHandle(windows.Handle(f.Fd()), &info); err != nil { + return err + } + if info.NumberOfLinks != 1 { + return errf("unknown") + } + return nil +} + +func replaceFile(from, to string) error { + source, err := windows.UTF16PtrFromString(from) + if err != nil { + return err + } + target, err := windows.UTF16PtrFromString(to) + if err != nil { + return err + } + return windows.MoveFileEx(source, target, windows.MOVEFILE_REPLACE_EXISTING|windows.MOVEFILE_WRITE_THROUGH) +} + +func syncDirectory(string) error { return nil } diff --git a/internal/dotevents/identity.go b/internal/dotevents/identity.go new file mode 100644 index 0000000..69cac14 --- /dev/null +++ b/internal/dotevents/identity.go @@ -0,0 +1,13 @@ +package dotevents + +import ( + "crypto/sha256" + "encoding/hex" +) + +// The occurrence eventId identifies the source notice. Standard Webhooks IDs +// identify a delivery to one subscription and remain stable across retries. +func deliveryID(subscriptionID, eventID string) string { + sum := sha256.Sum256([]byte(deliveryKey(subscriptionID, eventID))) + return "evt_" + hex.EncodeToString(sum[:]) +} diff --git a/internal/dotevents/lifecycle_test.go b/internal/dotevents/lifecycle_test.go new file mode 100644 index 0000000..652ef7f --- /dev/null +++ b/internal/dotevents/lifecycle_test.go @@ -0,0 +1,236 @@ +package dotevents + +import ( + "bytes" + "context" + "crypto/hmac" + "crypto/sha256" + "encoding/base64" + "encoding/json" + "io" + "net/http" + "os" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" +) + +func echoVerification(w http.ResponseWriter, r *http.Request) bool { + var v map[string]json.RawMessage + b, _ := io.ReadAll(r.Body) + _ = json.Unmarshal(b, &v) + if string(v["type"]) == `"verification"` { + var c string + _ = json.Unmarshal(v["challenge"], &c) + _ = json.NewEncoder(w).Encode(map[string]string{"challenge": c}) + return true + } + return false +} +func TestVerificationGenerationAndPause(t *testing.T) { + for _, action := range []string{"refresh", "unsubscribe", "pause"} { + t.Run(action, func(t *testing.T) { + entered := make(chan struct{}) + release := make(chan struct{}) + var calls atomic.Int32 + m, _, close, _ := withFixtureServerURL(t, func(w http.ResponseWriter, r *http.Request) { + if calls.Add(1) == 1 { + closeChannel(entered) + <-release + } + echoVerification(w, r) + }) + defer close() + p := testParams("alice") + done := make(chan error, 1) + go func() { _, e := m.Subscribe(context.Background(), "alice", p); done <- e }() + <-entered + switch action { + case "refresh": + p.Delivery.Secret = "whsec_" + base64.StdEncoding.EncodeToString(bytes.Repeat([]byte("x"), 32)) + if _, e := m.Subscribe(context.Background(), "alice", p); e != nil { + t.Fatal(e) + } + case "unsubscribe": + p.Delivery.Secret = "" + if e := m.Unsubscribe(context.Background(), "alice", p); e != nil { + t.Fatal(e) + } + case "pause": + if e := m.SetEnabled(false); e != nil { + t.Fatal(e) + } + if e := m.SetEnabled(true); e != nil { + t.Fatal(e) + } + } + closeChannel(release) + if e := <-done; e == nil { + t.Fatal("old verification accepted") + } + s, _ := m.Status("alice", "alice") + if action != "refresh" && s.CallbackVerified { + t.Fatal("revoked verification revived") + } + }) + } +} +func closeChannel(c chan struct{}) { close(c) } +func TestRetriesPersistOneAttemptAndTerminal(t *testing.T) { + for _, status := range []int{500, 429, 410, 413, 400, 302} { + t.Run(http.StatusText(status), func(t *testing.T) { + var sends atomic.Int32 + var ids []string + var bodies [][]byte + m, dir, close, _ := withFixtureServerURL(t, func(w http.ResponseWriter, r *http.Request) { + b, _ := io.ReadAll(r.Body) + var v map[string]json.RawMessage + _ = json.Unmarshal(b, &v) + if string(v["type"]) == `"verification"` { + var c string + _ = json.Unmarshal(v["challenge"], &c) + _ = json.NewEncoder(w).Encode(map[string]string{"challenge": c}) + return + } + sends.Add(1) + ids = append(ids, r.Header.Get("webhook-id")) + bodies = append(bodies, b) + w.WriteHeader(status) + }) + defer close() + if _, e := m.Subscribe(context.Background(), "alice", testParams("alice")); e != nil { + t.Fatal(e) + } + n := Notice{"evt.retry", "alice", "job.retry", "answer"} + e := m.Deliver(context.Background(), "alice", "alice", n) + if e == nil || sends.Load() != 1 { + t.Fatalf("attempt %v %d", e, sends.Load()) + } + s, _ := m.Status("alice", "alice") + if s.Attempts != 1 { + t.Fatal("failure attempt not persisted") + } + retry := status == 500 || status == 429 + if retry && asEventsError(e).RetryAfter != time.Second { + t.Fatalf("retry %v", e) + } + if !retry && asEventsError(e).RetryAfter != 0 { + t.Fatal("terminal retry") + } + reopened, e := Open(dir, testKey, m.delivery) + if e != nil { + t.Fatal(e) + } + for attempt := 2; attempt <= 5; attempt++ { + if retry { + _ = m.store.withLock(false, true, func(st *persistenceState) error { + for _, r := range st.Deliveries { + r.NextAttempt = time.Time{} + } + return nil + }) + } + _ = reopened.Deliver(context.Background(), "alice", "alice", n) + } + want := int32(1) + if retry { + want = 4 + } + if sends.Load() != want { + t.Fatalf("unbounded sends %d", sends.Load()) + } + for i := range ids { + if ids[i] != deliveryID(subscriptionID("alice", testParams("alice")), n.EventID) || !bytes.Equal(bodies[0], bodies[i]) { + t.Fatal("retry changed identity or body") + } + } + }) + } +} +func TestUnsubscribeExpiryEncryptionAndSignatures(t *testing.T) { + var checked atomic.Bool + m, dir, close, _ := withFixtureServerURL(t, func(w http.ResponseWriter, r *http.Request) { + b, _ := io.ReadAll(r.Body) + var v map[string]json.RawMessage + _ = json.Unmarshal(b, &v) + key, _ := decodeSecret(testSecret()) + mac := hmac.New(sha256.New, key) + mac.Write([]byte(r.Header.Get("webhook-id") + "." + r.Header.Get("webhook-timestamp") + "." + string(b))) + want := "v1," + base64.StdEncoding.EncodeToString(mac.Sum(nil)) + if r.Header.Get("webhook-signature") != want { + t.Error("signature does not match exact transmitted bytes") + } + if string(v["type"]) == `"verification"` { + var c string + _ = json.Unmarshal(v["challenge"], &c) + _ = json.NewEncoder(w).Encode(map[string]string{"challenge": c}) + return + } + checked.Store(true) + w.WriteHeader(204) + }) + defer close() + p := testParams("alice") + p.TTL = json.RawMessage(`null`) + sub, e := m.Subscribe(context.Background(), "alice", p) + if e != nil { + t.Fatal(e) + } + if time.Until(sub.ExpiresAt) > time.Hour { + t.Fatal("unbounded null ttl") + } + if e = m.Deliver(context.Background(), "alice", "alice", Notice{"evt.signed", "alice", "job.signed", "task"}); e != nil { + t.Fatal(e) + } + if !checked.Load() { + t.Fatal("no signed event") + } + disk, _ := os.ReadFile(filepath.Join(dir, stateFileName)) + if bytes.Contains(disk, []byte(p.Delivery.Secret)) || bytes.Contains(disk, []byte("opaque=ok")) { + t.Fatal("plaintext persisted") + } + _ = m.store.withLock(false, true, func(st *persistenceState) error { + for _, s := range st.Subscriptions { + s.ExpiresAt = time.Now().Add(-time.Second) + } + return nil + }) + if e = m.Deliver(context.Background(), "alice", "alice", Notice{"evt.expired", "alice", "job.signed", "task"}); e == nil { + t.Fatal("expired sent") + } + p.Delivery.Secret = "" + for i := 0; i < 2; i++ { + if e = m.Unsubscribe(context.Background(), "alice", p); e != nil { + t.Fatal(e) + } + } + wrong, _ := Open(dir, bytes.Repeat([]byte("x"), 32), nil) + if wrong != nil { + t.Fatal("wrong key accepted") + } +} +func TestRedirectNotFollowedAndMismatch(t *testing.T) { + for _, kind := range []string{"redirect", "mismatch", "raw"} { + t.Run(kind, func(t *testing.T) { + m, _, close, _ := withFixtureServerURL(t, func(w http.ResponseWriter, r *http.Request) { + if kind == "redirect" { + w.Header().Set("Location", "https://127.0.0.1/") + w.WriteHeader(302) + return + } + if kind == "raw" { + _, _ = w.Write([]byte("not-json")) + return + } + _, _ = w.Write([]byte(`{"challenge":"wrong"}`)) + }) + defer close() + _, e := m.Subscribe(context.Background(), "alice", testParams("alice")) + if e == nil || strings.Contains(e.Error(), "opaque") { + t.Fatalf("verification error %v", e) + } + }) + } +} diff --git a/internal/dotevents/manager.go b/internal/dotevents/manager.go new file mode 100644 index 0000000..561bb24 --- /dev/null +++ b/internal/dotevents/manager.go @@ -0,0 +1,265 @@ +package dotevents + +import ( + "context" + "sort" + "time" +) + +type Manager struct { + store *store + delivery *Delivery +} + +func Open(dir string, key []byte, delivery *Delivery) (*Manager, error) { + st, e := openStore(dir, key) + if e != nil { + return nil, e + } + if delivery == nil { + delivery = &Delivery{} + } + m := &Manager{st, delivery} + if e = st.withLock(true, true, func(*persistenceState) error { return nil }); e != nil { + return nil, e + } + return m, nil +} +func (m *Manager) SetEnabled(enabled bool) error { + return m.store.withLock(false, true, func(st *persistenceState) error { + st.Enabled = enabled + if !enabled { + for _, s := range st.Subscriptions { + s.CallbackVerified = false + s.Generation = "" + } + } + return nil + }) +} +func (m *Manager) Subscribe(ctx context.Context, principal string, p Params) (Subscription, error) { + if e := validateParams(principal, p); e != nil { + return Subscription{}, e + } + ttl, _ := parseTTL(p.TTL) + id := subscriptionID(principal, p) + gen, e := randomToken(16) + if e != nil { + return Subscription{}, e + } + now := time.Now().UTC() + rec := &subscriptionRecord{ID: id, Generation: gen, Principal: principal, Params: p, Secret: p.Delivery.Secret, ExpiresAt: now.Add(time.Duration(ttl) * time.Millisecond), CreatedAt: now, UpdatedAt: now} + if e = m.store.withLock(false, true, func(st *persistenceState) error { + if !st.Enabled { + return errf("disabled") + } + for k, s := range st.Subscriptions { + if !s.ExpiresAt.After(now) { + delete(st.Subscriptions, k) + } + } + old := st.Subscriptions[id] + if old == nil && len(st.Subscriptions) >= maxActiveSubs { + return errf("capacity") + } + if old != nil { + rec.CreatedAt = old.CreatedAt + if old.Secret != rec.Secret { + rec.PreviousSecret = old.Secret + rec.RotateUntil = now.Add(secretRotateGrace) + } else { + rec.PreviousSecret = old.PreviousSecret + rec.RotateUntil = old.RotateUntil + } + } + st.Subscriptions[id] = rec + return nil + }); e != nil { + return Subscription{}, e + } + if e = m.delivery.verify(ctx, rec); e != nil { + return Subscription{}, e + } + e = m.store.withLock(false, true, func(st *persistenceState) error { + if !st.Enabled { + return errf("disabled") + } + s := st.Subscriptions[id] + if s == nil || s.Generation != gen || !s.ExpiresAt.After(time.Now()) { + return errf("stale") + } + s.CallbackVerified = true + return nil + }) + if e != nil { + return Subscription{}, e + } + return Subscription{ID: id, ExpiresAt: rec.ExpiresAt, CallbackVerified: true}, nil +} +func (m *Manager) Unsubscribe(ctx context.Context, principal string, p Params) error { + if ctx.Err() != nil { + return errf("cancelled") + } + if validateID(principal) != nil || p.Arguments.QueueID != principal || p.Name != EventName || p.Delivery.Mode != "webhook" || validateCallbackURL(p.Delivery.URL) != nil { + return errf("unknown") + } + return m.store.withLock(false, true, func(st *persistenceState) error { delete(st.Subscriptions, subscriptionID(principal, p)); return nil }) +} +func (m *Manager) List(principal string) []Definition { + if validateID(principal) != nil { + return nil + } + return []Definition{{Name: EventName, Description: "A queued task is available; retrieve it using authorized queue tools."}} +} +func (m *Manager) Deliver(ctx context.Context, principal, queueID string, n Notice) error { + if validateID(principal) != nil || principal != queueID || n.QueueID != queueID || validateID(n.EventID) != nil || validateID(n.JobID) != nil || (n.Mode != "answer" && n.Mode != "task") { + return errf("unknown") + } + var ids []string + if e := m.store.withLock(false, false, func(st *persistenceState) error { + if !st.Enabled { + return errf("disabled") + } + for id, s := range st.Subscriptions { + if s.Principal == principal && s.CallbackVerified && s.ExpiresAt.After(time.Now()) { + ids = append(ids, id) + } + } + return nil + }); e != nil { + return e + } + if len(ids) == 0 { + return errf("notfound") + } + sort.Strings(ids) + var result error + for _, id := range ids { + if e := m.deliverOne(ctx, principal, queueID, id, n); e != nil { + if result == nil || asEventsError(e).RetryAfter > 0 { + result = e + } + } + } + return result +} +func (m *Manager) deliverOne(ctx context.Context, principal, q, id string, n Notice) error { + var sub *subscriptionRecord + var body []byte + var attempt int + var skip bool + var outcome error + key := deliveryKey(id, n.EventID) + e := m.store.withLock(false, true, func(st *persistenceState) error { + if !st.Enabled { + return errf("disabled") + } + s := st.Subscriptions[id] + if s == nil || !s.CallbackVerified || !s.ExpiresAt.After(time.Now()) { + return errf("stale") + } + r := st.Deliveries[key] + now := time.Now().UTC() + if r != nil { + if r.Acknowledged || r.Terminal { + skip = true + return nil + } + if r.InFlightUntil.After(now) { + skip = true + outcome = retryErr("in_flight", time.Second) + return nil + } + if r.NextAttempt.After(now) { + skip = true + outcome = retryErr("retry", r.NextAttempt.Sub(now)) + return nil + } + } + if r == nil { + if len(st.Deliveries) >= maxDeliveryRecords { + return errf("capacity") + } + r = &deliveryRecord{SubscriptionID: id, EventID: n.EventID, Body: eventBodyBytes(n)} + st.Deliveries[key] = r + } + if r.Attempts >= 4 { + r.Terminal = true + skip = true + outcome = errf("exhausted") + return nil + } + r.Attempts++ + r.InFlightUntil = now.Add(maxHTTPTimeout + time.Second) + r.UpdatedAt = now + attempt = r.Attempts + body = append([]byte(nil), r.Body...) + cp := *s + sub = &cp + return nil + }) + if e != nil { + return e + } + if skip { + return outcome + } + code, sendErr := m.delivery.deliverBytes(ctx, sub, n.EventID, body) + e = m.store.withLock(false, true, func(st *persistenceState) error { + if !st.Enabled { + return errf("disabled") + } + s := st.Subscriptions[id] + if s == nil || s.Generation != sub.Generation || !s.CallbackVerified || !s.ExpiresAt.After(time.Now()) { + return errf("stale") + } + r := st.Deliveries[key] + if r == nil || r.Attempts != attempt { + return errf("stale") + } + r.InFlightUntil = time.Time{} + r.LastCode = code + r.UpdatedAt = time.Now().UTC() + r.Acknowledged = sendErr == nil + if sendErr != nil { + c := asEventsError(sendErr).Code + if (c == "network" || c == "server" || c == "rate_limited") && attempt <= len(retrySchedule) { + delay := retrySchedule[attempt-1] + r.NextAttempt = time.Now().Add(delay) + outcome = retryErr(c, delay) + } else { + r.Terminal = true + outcome = sendErr + } + } + st.QueueStatus[queueKey(principal, q)] = &queueStatus{Attempts: r.Attempts, Acknowledged: r.Acknowledged, LastCode: code, LastEventID: n.EventID} + return nil + }) + if e != nil { + return e + } + return outcome +} +func (m *Manager) Status(principal, q string) (Status, error) { + if validateID(principal) != nil || principal != q { + return Status{}, errf("unknown") + } + var out Status + e := m.store.withLock(false, false, func(st *persistenceState) error { + if st.Enabled { + for _, s := range st.Subscriptions { + if s.Principal == principal && s.ExpiresAt.After(time.Now()) { + out.Subscribed = true + out.CallbackVerified = out.CallbackVerified || s.CallbackVerified + } + } + } + if r := st.QueueStatus[queueKey(principal, q)]; r != nil { + out.Attempts = r.Attempts + out.Acknowledged = r.Acknowledged + out.LastCode = r.LastCode + } + return nil + }) + return out, e +} diff --git a/internal/dotevents/multiple_test.go b/internal/dotevents/multiple_test.go new file mode 100644 index 0000000..aa38d7b --- /dev/null +++ b/internal/dotevents/multiple_test.go @@ -0,0 +1,85 @@ +package dotevents + +import ( + "context" + "encoding/json" + "io" + "net/http" + "sync/atomic" + "testing" + "time" +) + +func TestMultipleSubscriptionsAndRevokeDuringDelivery(t *testing.T) { + var sends atomic.Int32 + entered := make(chan struct{}) + release := make(chan struct{}) + var block atomic.Bool + m, _, stop, _ := withFixtureServerURL(t, func(w http.ResponseWriter, r *http.Request) { + b, _ := io.ReadAll(r.Body) + var v map[string]json.RawMessage + _ = json.Unmarshal(b, &v) + if string(v["type"]) == `"verification"` { + var c string + _ = json.Unmarshal(v["challenge"], &c) + _ = json.NewEncoder(w).Encode(map[string]string{"challenge": c}) + return + } + sends.Add(1) + if block.Load() { + close(entered) + <-release + } + w.WriteHeader(204) + }) + defer stop() + p := testParams("alice") + if _, e := m.Subscribe(context.Background(), "alice", p); e != nil { + t.Fatal(e) + } + second := p + second.Delivery.URL += "&second=1" + if _, e := m.Subscribe(context.Background(), "alice", second); e != nil { + t.Fatal(e) + } + if e := m.Deliver(context.Background(), "alice", "alice", Notice{"evt.multi", "alice", "job.multi", "answer"}); e != nil { + t.Fatal(e) + } + if sends.Load() != 2 { + t.Fatalf("sent to %d subscriptions", sends.Load()) + } + if e := m.Unsubscribe(context.Background(), "alice", second); e != nil { + t.Fatal(e) + } + block.Store(true) + done := make(chan error, 1) + go func() { + done <- m.Deliver(context.Background(), "alice", "alice", Notice{"evt.block", "alice", "job.multi", "task"}) + }() + <-entered + if e := m.SetEnabled(false); e != nil { + t.Fatal(e) + } + if e := m.SetEnabled(true); e != nil { + t.Fatal(e) + } + close(release) + if e := <-done; e == nil { + t.Fatal("late callback acknowledged after revoke") + } + s, _ := m.Status("alice", "alice") + if s.CallbackVerified { + t.Fatal("pause revived verification") + } +} +func TestTTLBoundsAndRotationExpiry(t *testing.T) { + for _, raw := range []string{`"60000"`, `true`, `-1`, `86400001`, `{}`, `[]`} { + if _, e := parseTTL(json.RawMessage(raw)); e == nil { + t.Errorf("TTL accepted %s", raw) + } + } + sub := &subscriptionRecord{Secret: testSecret(), PreviousSecret: testSecret(), RotateUntil: time.Now().Add(-time.Second)} + if len(signPayload(sub, "id", []byte("中文"), 1)) != 1 { + t.Fatal("old secret signed after grace") + } +} diff --git a/internal/dotevents/params.go b/internal/dotevents/params.go new file mode 100644 index 0000000..843951b --- /dev/null +++ b/internal/dotevents/params.go @@ -0,0 +1,112 @@ +package dotevents + +import ( + "bytes" + "encoding/json" + "io" + "unicode/utf8" +) + +// UnmarshalJSON rejects ambiguous duplicate keys and unknown fields before the +// integration adapter can bind the authenticated principal. +func (p *Params) UnmarshalJSON(data []byte) error { + if len(data) > maxParamsBytes || !utf8.Valid(data) { + return errf("unknown") + } + d := json.NewDecoder(bytes.NewReader(data)) + if e := uniqueJSON(d, 0); e != nil { + return e + } + if _, e := d.Token(); e != io.EOF { + return errf("unknown") + } + var obj map[string]json.RawMessage + if json.Unmarshal(data, &obj) != nil || obj == nil { + return errf("unknown") + } + for k, v := range obj { + switch k { + case "name", "arguments", "delivery", "cursor", "ttlMs": + default: + return errf("unknown") + } + if k != "cursor" && k != "ttlMs" && bytes.Equal(bytes.TrimSpace(v), []byte("null")) { + return errf("unknown") + } + } + for _, k := range []string{"name", "arguments", "delivery"} { + if _, ok := obj[k]; !ok { + return errf("unknown") + } + } + for k, fields := range map[string][]string{"arguments": {"queue_id"}, "delivery": {"mode", "url", "secret"}} { + var nested map[string]json.RawMessage + if json.Unmarshal(obj[k], &nested) != nil || nested == nil { + return errf("unknown") + } + for key, v := range nested { + allowed := false + for _, f := range fields { + if f == key { + allowed = true + } + } + if !allowed || bytes.Equal(bytes.TrimSpace(v), []byte("null")) { + return errf("unknown") + } + } + } + type plain Params + var out plain + decoder := json.NewDecoder(bytes.NewReader(data)) + decoder.DisallowUnknownFields() + if decoder.Decode(&out) != nil { + return errf("unknown") + } + *p = Params(out) + return nil +} +func uniqueJSON(d *json.Decoder, depth int) error { + if depth > 16 { + return errf("unknown") + } + tok, e := d.Token() + if e != nil { + return errf("unknown") + } + delim, ok := tok.(json.Delim) + if !ok { + return nil + } + switch delim { + case '{': + seen := map[string]bool{} + for d.More() { + k, e := d.Token() + if e != nil { + return errf("unknown") + } + s, ok := k.(string) + if !ok || seen[s] { + return errf("unknown") + } + seen[s] = true + if e = uniqueJSON(d, depth+1); e != nil { + return e + } + } + case '[': + for d.More() { + if e = uniqueJSON(d, depth+1); e != nil { + return e + } + } + default: + return errf("unknown") + } + end, e := d.Token() + if e != nil || (delim == '{' && end != json.Delim('}')) || (delim == '[' && end != json.Delim(']')) { + return errf("unknown") + } + return nil +} diff --git a/internal/dotevents/security_test.go b/internal/dotevents/security_test.go new file mode 100644 index 0000000..9cc3937 --- /dev/null +++ b/internal/dotevents/security_test.go @@ -0,0 +1,68 @@ +package dotevents + +import ( + "context" + "errors" + "net" + "net/netip" + "testing" +) + +func TestReservedGuardCoverage(t *testing.T) { + for _, s := range []string{"0.1.2.3", "192.0.0.8", "192.88.99.1", "198.19.0.1", "2001::1", "2002:a00:1::", "3fff::1", "4000::1", "203.0.113.10"} { + if publicIP(netip.MustParseAddr(s)) { + t.Errorf("reserved IP accepted %s", s) + } + } + for _, s := range []string{"8.8.8.8", "1.1.1.1", "2606:4700:4700::1111"} { + if !publicIP(netip.MustParseAddr(s)) { + t.Errorf("public IP rejected %s", s) + } + } +} +func TestProductionDialGuardRebindingAndAllAnswers(t *testing.T) { + calls, dials := 0, 0 + resolve := func(context.Context, string) ([]net.IPAddr, error) { + calls++ + if calls == 1 { + return []net.IPAddr{{IP: net.ParseIP("8.8.8.8")}}, nil + } + return []net.IPAddr{{IP: net.ParseIP("8.8.8.8")}, {IP: net.ParseIP("127.0.0.1")}}, nil + } + dial := func(_ context.Context, _, address string) (net.Conn, error) { + dials++ + if address != "8.8.8.8:443" { + t.Errorf("dial not pinned: %s", address) + } + return nil, errors.New("offline") + } + tr := publicTransport(resolve, dial) + if tr.Proxy != nil { + t.Fatal("environment proxy enabled") + } + _, _ = tr.DialContext(context.Background(), "tcp", "callback.example:443") + _, _ = tr.DialContext(context.Background(), "tcp", "callback.example:443") + if calls != 2 || dials != 1 { + t.Fatalf("resolver=%d dial=%d", calls, dials) + } + for _, s := range []string{"127.0.0.1:443", "[::1]:443", "169.254.169.254:443", "203.0.113.10:443"} { + _, _ = tr.DialContext(context.Background(), "tcp", s) + } + if dials != 1 { + t.Fatal("private literal reached socket") + } +} +func TestStrictParamsJSON(t *testing.T) { + for _, s := range []string{ + `{"name":"dot.task.available","name":"other"}`, + `{"name":"dot.task.available","arguments":{"queue_id":"alice","extra":true}}`, + `{"name":"dot.task.available","arguments":{"queue_id":null}}`, + `{"Name":"dot.task.available"}`, + `{"name":"dot.task.available","delivery":{"mode":"webhook","url":"https://callback.example","secret":"x","extra":1}}`, + } { + var p Params + if e := p.UnmarshalJSON([]byte(s)); e == nil { + t.Errorf("accepted invalid schema %s", s) + } + } +} diff --git a/internal/dotevents/state.go b/internal/dotevents/state.go new file mode 100644 index 0000000..f543f6a --- /dev/null +++ b/internal/dotevents/state.go @@ -0,0 +1,170 @@ +package dotevents + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "time" +) + +const ( + stateVersion = 1 + markerName = "dot_events.initialized" + lockName = "dot_events.lock" + stateFileName = "events.state.de" + maxStateBytes = 8 * 1024 * 1024 +) + +type subscriptionRecord struct { + Generation string `json:"generation"` + ID string `json:"id"` + Principal string `json:"principal"` + Params Params `json:"params"` + Secret string `json:"secret"` + PreviousSecret string `json:"previous_secret,omitempty"` + RotateUntil time.Time `json:"rotate_until,omitempty"` + ExpiresAt time.Time `json:"expires_at"` + CallbackVerified bool `json:"callback_verified"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` +} + +type deliveryRecord struct { + Body []byte `json:"body"` + InFlightUntil time.Time `json:"in_flight_until"` + NextAttempt time.Time `json:"next_attempt"` + Terminal bool `json:"terminal"` + SubscriptionID string `json:"subscription_id"` + EventID string `json:"event_id"` + Attempts int `json:"attempts"` + Acknowledged bool `json:"acknowledged"` + LastCode string `json:"last_code,omitempty"` + UpdatedAt time.Time `json:"updated_at"` +} + +type queueStatus struct { + Attempts int `json:"attempts"` + Acknowledged bool `json:"acknowledged"` + LastCode string `json:"last_code,omitempty"` + LastEventID string `json:"last_event_id,omitempty"` +} + +type persistenceState struct { + StateVersion int `json:"state_version"` + Enabled bool `json:"enabled"` + Subscriptions map[string]*subscriptionRecord `json:"subscriptions"` + Deliveries map[string]*deliveryRecord `json:"deliveries"` + QueueStatus map[string]*queueStatus `json:"queue_status"` +} + +func newPersistenceState() *persistenceState { + return &persistenceState{ + StateVersion: stateVersion, + Enabled: true, + Subscriptions: map[string]*subscriptionRecord{}, + Deliveries: map[string]*deliveryRecord{}, + QueueStatus: map[string]*queueStatus{}, + } +} + +func parsePersistenceState(data []byte) (*persistenceState, error) { + if len(data) > maxStateBytes || !json.Valid(data) { + return nil, errf("corrupt") + } + var st persistenceState + if json.Unmarshal(data, &st) != nil { + return nil, errf("corrupt") + } + if st.StateVersion != stateVersion || st.Subscriptions == nil || st.Deliveries == nil || st.QueueStatus == nil || len(st.Subscriptions) > maxActiveSubs || len(st.Deliveries) > maxDeliveryRecords || len(st.QueueStatus) > maxDeliveryRecords { + return nil, errf("corrupt") + } + for id, sub := range st.Subscriptions { + if sub == nil || sub.ID != id || sub.ID != subscriptionID(sub.Principal, sub.Params) || validateID(sub.Principal) != nil || validateParams(sub.Principal, sub.Params) != nil || validateSecret(sub.Secret) != nil || sub.Secret != sub.Params.Delivery.Secret || (sub.PreviousSecret != "" && validateSecret(sub.PreviousSecret) != nil) { + return nil, errf("corrupt") + } + } + for key, rec := range st.Deliveries { + if rec == nil || deliveryKey(rec.SubscriptionID, rec.EventID) != key || validateID(rec.SubscriptionID) != nil || validateID(rec.EventID) != nil || rec.Attempts < 1 || rec.Attempts > 4 || len(rec.Body) > maxEventBodyBytes || !json.Valid(rec.Body) { + return nil, errf("corrupt") + } + } + for key, stRec := range st.QueueStatus { + parts := splitQueueKey(key) + if validateID(parts[0]) != nil || validateID(parts[1]) != nil { + return nil, errf("corrupt") + } + if stRec == nil { + return nil, errf("corrupt") + } + } + return &st, nil +} + +func subscriptionID(principal string, p Params) string { + canon := struct { + Principal string `json:"principal"` + Name string `json:"name"` + Arguments Arguments `json:"arguments"` + Delivery struct { + Mode string `json:"mode"` + URL string `json:"url"` + } `json:"delivery"` + }{ + Principal: principal, + Name: p.Name, + Arguments: p.Arguments, + Delivery: struct { + Mode string `json:"mode"` + URL string `json:"url"` + }{Mode: p.Delivery.Mode, URL: p.Delivery.URL}, + } + sum := sha256.Sum256(mustMarshal(canon)) + return hex.EncodeToString(sum[:]) +} + +func deliveryKey(subscriptionID, eventID string) string { + return subscriptionID + "\x00" + eventID +} + +func queueKey(principal, queueID string) string { + return principal + "\x00" + queueID +} + +func splitQueueKey(key string) [2]string { + parts := [2]string{"", ""} + for i := 0; i < len(key); i++ { + if key[i] == 0 { + parts[0] = key[:i] + parts[1] = key[i+1:] + break + } + } + return parts +} + +func activeSubscriptions(st *persistenceState, now time.Time) int { + n := 0 + for _, sub := range st.Subscriptions { + if sub != nil && sub.ExpiresAt.After(now) { + n++ + } + } + return n +} + +func pruneDeliveries(st *persistenceState) { + if len(st.Deliveries) <= maxDeliveryRecords { + return + } + var oldest string + var oldestAt time.Time + for key, rec := range st.Deliveries { + if oldest == "" || rec.UpdatedAt.Before(oldestAt) { + oldest = key + oldestAt = rec.UpdatedAt + } + } + if oldest != "" { + delete(st.Deliveries, oldest) + } +} diff --git a/internal/dotevents/store.go b/internal/dotevents/store.go new file mode 100644 index 0000000..9d9c181 --- /dev/null +++ b/internal/dotevents/store.go @@ -0,0 +1,229 @@ +package dotevents + +import ( + "context" + "encoding/json" + "io" + "os" + "path/filepath" + "strings" + "time" + "unicode" + + "github.com/gofrs/flock" +) + +type store struct { + dir string + key []byte +} + +func openStore(dir string, key []byte) (*store, error) { + if len(key) != 32 { + return nil, errf("unknown") + } + if !filepath.IsAbs(dir) || len(dir) > 32768 || strings.ContainsFunc(dir, unicode.IsControl) { + return nil, errf("unknown") + } + return &store{dir: dir, key: append([]byte(nil), key...)}, nil +} + +func (s *store) withLock(allowCreate bool, mutate bool, fn func(*persistenceState) error) error { + if err := s.checkDir(allowCreate); err != nil { + return err + } + lockPath := filepath.Join(s.dir, lockName) + if err := regularFile(lockPath); err != nil { + return wrapUnknown(err) + } + f, err := os.OpenFile(lockPath, os.O_CREATE|os.O_RDWR, 0600) + if err != nil { + return errf("unknown") + } + info, statErr := f.Stat() + if statErr != nil || rejectHardlink(f, info) != nil { + f.Close() + return errf("unknown") + } + if f.Close() != nil || privatePath(lockPath, false) != nil { + return errf("unknown") + } + lock := flock.New(lockPath) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + ok, err := lock.TryLockContext(ctx, 10*time.Millisecond) + if err != nil || !ok { + return errf("unknown") + } + defer lock.Unlock() + + st, err := s.loadState(allowCreate) + if err != nil { + return err + } + if st == nil { + st = newPersistenceState() + } + if mutate { + if err := fn(st); err != nil { + return err + } + return s.saveState(st) + } + return fn(st) +} + +func (s *store) loadState(allowCreate bool) (*persistenceState, error) { + marker := filepath.Join(s.dir, markerName) + _, markerErr := os.Lstat(marker) + statePath := filepath.Join(s.dir, stateFileName) + _, stateErr := os.Lstat(statePath) + if os.IsNotExist(markerErr) && os.IsNotExist(stateErr) { + if allowCreate { + return nil, nil + } + return nil, errf("corrupt") + } + if markerErr != nil || stateErr != nil { + if os.IsNotExist(stateErr) && markerErr == nil { + return nil, errf("corrupt") + } + return nil, errf("corrupt") + } + if err := regularFile(statePath); err != nil { + return nil, errf("corrupt") + } + f, err := os.Open(statePath) + if err != nil { + return nil, errf("corrupt") + } + defer f.Close() + info, err := f.Stat() + if err != nil || !info.Mode().IsRegular() || rejectHardlink(f, info) != nil { + return nil, errf("corrupt") + } + if info.Size() > maxStateBytes { + return nil, errf("corrupt") + } + data, err := io.ReadAll(io.LimitReader(f, maxStateBytes+1)) + if err != nil { + return nil, errf("corrupt") + } + plain, err := openEnvelope(s.key, data) + if err != nil { + return nil, err + } + return parsePersistenceState(plain) +} + +func (s *store) saveState(st *persistenceState) error { + if st.StateVersion != stateVersion { + return errf("corrupt") + } + data, err := jsonMarshal(st) + if err != nil || len(data) > maxStateBytes { + return errf("unknown") + } + sealed, err := seal(s.key, data) + if err != nil { + return err + } + statePath := filepath.Join(s.dir, stateFileName) + if err := regularFile(statePath); err != nil { + return errf("unknown") + } + f, err := os.CreateTemp(s.dir, ".dot-events-*") + if err != nil { + return errf("unknown") + } + tmp := f.Name() + defer os.Remove(tmp) + if privatePath(tmp, false) != nil { + f.Close() + return errf("unknown") + } + n, writeErr := f.Write(sealed) + syncErr := f.Sync() + closeErr := f.Close() + if writeErr != nil || n != len(sealed) || syncErr != nil || closeErr != nil { + return errf("unknown") + } + if regularFile(statePath) != nil || replaceFile(tmp, statePath) != nil || syncDirectory(s.dir) != nil { + return errf("unknown") + } + return s.ensureMarker() +} + +func (s *store) ensureMarker() error { + path := filepath.Join(s.dir, markerName) + if err := regularFile(path); err != nil { + return errf("unknown") + } + f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600) + if os.IsExist(err) { + f, err = os.Open(path) + if err != nil { + return errf("unknown") + } + info, statErr := f.Stat() + if statErr != nil || rejectHardlink(f, info) != nil { + f.Close() + return errf("unknown") + } + if f.Close() != nil || privatePath(path, false) != nil { + return errf("unknown") + } + return nil + } + if err != nil { + return errf("unknown") + } + if privatePath(path, false) != nil { + f.Close() + return errf("unknown") + } + _, writeErr := f.WriteString("1\n") + syncErr := f.Sync() + closeErr := f.Close() + if writeErr != nil || syncErr != nil || closeErr != nil || syncDirectory(s.dir) != nil { + return errf("unknown") + } + return nil +} + +func (s *store) checkDir(create bool) error { + if !filepath.IsAbs(s.dir) || len(s.dir) > 32768 || strings.ContainsFunc(s.dir, unicode.IsControl) { + return errf("unknown") + } + if create { + if os.MkdirAll(s.dir, 0700) != nil { + return errf("unknown") + } + } + info, err := os.Lstat(s.dir) + if os.IsNotExist(err) { + return errf("corrupt") + } + if err != nil || !info.IsDir() || rejectLink(info) != nil { + return errf("unknown") + } + if create && privatePath(s.dir, true) != nil { + return errf("unknown") + } + return nil +} + +func regularFile(path string) error { + info, err := os.Lstat(path) + if os.IsNotExist(err) { + return nil + } + if err != nil || !info.Mode().IsRegular() || rejectLink(info) != nil { + return errf("unknown") + } + return nil +} + +func jsonMarshal(v any) ([]byte, error) { + return json.Marshal(v) +} diff --git a/internal/dotevents/store_test.go b/internal/dotevents/store_test.go new file mode 100644 index 0000000..4dfba49 --- /dev/null +++ b/internal/dotevents/store_test.go @@ -0,0 +1,62 @@ +package dotevents + +import ( + "os" + "os/exec" + "path/filepath" + "testing" +) + +func TestProcessLockHelper(t *testing.T) { + if os.Getenv("DOT_EVENTS_TEST_HELPER") != "1" { + return + } + m, e := Open(os.Getenv("DOT_EVENTS_TEST_DIR"), testKey, nil) + if e != nil { + t.Fatal(e) + } + for i := 0; i < 12; i++ { + if e = m.SetEnabled(i%2 == 0); e != nil { + t.Fatal(e) + } + } +} +func TestCrossProcessLockAndMissingState(t *testing.T) { + m, dir := openTestManager(t) + exe, e := os.Executable() + if e != nil { + t.Fatal(e) + } + cmds := make([]*exec.Cmd, 3) + for i := range cmds { + c := exec.Command(exe, "-test.run=^TestProcessLockHelper$") + c.Env = append(os.Environ(), "DOT_EVENTS_TEST_HELPER=1", "DOT_EVENTS_TEST_DIR="+dir) + cmds[i] = c + if e = c.Start(); e != nil { + t.Fatal(e) + } + } + for _, c := range cmds { + if e = c.Wait(); e != nil { + t.Fatal(e) + } + } + if _, e = m.Status("alice", "alice"); e != nil { + t.Fatal(e) + } + if e = os.Remove(filepath.Join(dir, stateFileName)); e != nil { + t.Fatal(e) + } + if _, e = m.Status("alice", "alice"); e == nil { + t.Fatal("missing state reset") + } +} +func TestRejectHardlinkState(t *testing.T) { + m, dir := openTestManager(t) + if e := os.Link(filepath.Join(dir, stateFileName), filepath.Join(dir, "alias")); e != nil { + t.Skip(e) + } + if _, e := m.Status("alice", "alice"); e == nil { + t.Fatal("hardlinked state accepted") + } +} diff --git a/internal/dotevents/testutil_test.go b/internal/dotevents/testutil_test.go new file mode 100644 index 0000000..548bb7b --- /dev/null +++ b/internal/dotevents/testutil_test.go @@ -0,0 +1,56 @@ +package dotevents + +import ( + "context" + "crypto/x509" + "encoding/base64" + "net" + "net/http" + "net/http/httptest" + "testing" + "time" +) + +const fixtureCallbackURL = "https://callback.example/hook?opaque=ok" + +var testKey = []byte("01234567890123456789012345678901") + +func openTestManager(t *testing.T) (*Manager, string) { + t.Helper() + dir := t.TempDir() + m, e := Open(dir, testKey, nil) + if e != nil { + t.Fatal(e) + } + return m, dir +} +func testSecret() string { + return "whsec_" + base64.StdEncoding.EncodeToString([]byte("01234567890123456789012345678901")) +} +func testParams(p string) Params { + return Params{Name: EventName, Arguments: Arguments{QueueID: p}, Delivery: Callback{Mode: "webhook", URL: fixtureCallbackURL, Secret: testSecret()}} +} +func withFixtureServerURL(t *testing.T, h http.HandlerFunc) (*Manager, string, func(), string) { + t.Helper() + s := httptest.NewTLSServer(h) + pool := x509.NewCertPool() + pool.AddCert(s.Certificate()) + tr := s.Client().Transport.(*http.Transport).Clone() + tr.Proxy = nil + tr.DisableKeepAlives = true + tr.TLSClientConfig.RootCAs = pool + // Explicit fixture routing and fixture certificate hostname, confined to tests. + tr.TLSClientConfig.ServerName = "example.com" + tr.DialContext = func(ctx context.Context, network, address string) (net.Conn, error) { + if address != "callback.example:443" { + return nil, errf("callback") + } + return (&net.Dialer{Timeout: time.Second}).DialContext(ctx, network, s.Listener.Addr().String()) + } + dir := t.TempDir() + m, e := Open(dir, testKey, &Delivery{client: &http.Client{Timeout: maxHTTPTimeout, Transport: tr, CheckRedirect: noRedirect}}) + if e != nil { + t.Fatal(e) + } + return m, dir, s.Close, fixtureCallbackURL +} diff --git a/internal/dotevents/types.go b/internal/dotevents/types.go new file mode 100644 index 0000000..31c2dd4 --- /dev/null +++ b/internal/dotevents/types.go @@ -0,0 +1,53 @@ +package dotevents + +import ( + "encoding/json" + "time" +) + +const EventName = "dot.task.available" + +type Arguments struct { + QueueID string `json:"queue_id"` +} + +type Callback struct { + Mode string `json:"mode"` + URL string `json:"url"` + Secret string `json:"secret"` +} + +type Params struct { + Name string `json:"name"` + Arguments Arguments `json:"arguments"` + Delivery Callback `json:"delivery"` + Cursor json.RawMessage `json:"cursor,omitempty"` + TTL json.RawMessage `json:"ttlMs,omitempty"` +} + +type Subscription struct { + ID string `json:"id"` + ExpiresAt time.Time `json:"expiresAt"` + CallbackVerified bool `json:"callbackVerified"` +} + +type Definition struct { + Name string `json:"name"` + Description string `json:"description"` +} + +type Notice struct { + EventID string + QueueID string + JobID string + Mode string +} + +type Status struct { + Subscribed bool `json:"subscribed"` + CallbackVerified bool `json:"callback_verified"` + DotReady bool `json:"dot_ready"` + Attempts int `json:"attempts"` + Acknowledged bool `json:"acknowledged"` + LastCode string `json:"last_code,omitempty"` +} diff --git a/internal/dotevents/validate.go b/internal/dotevents/validate.go new file mode 100644 index 0000000..1a26b5e --- /dev/null +++ b/internal/dotevents/validate.go @@ -0,0 +1,213 @@ +package dotevents + +import ( + "bytes" + "encoding/base64" + "encoding/json" + "net" + "net/netip" + "net/url" + "regexp" + "strings" + "unicode/utf8" +) + +const ( + maxCallbackURLBytes = 2048 + maxParamsBytes = 16 * 1024 + maxActiveSubs = 100 + maxDeliveryRecords = 1000 + minSecretBytes = 24 + maxSecretBytes = 64 + minTTL = 60 * 1000 + maxTTL = 24 * 60 * 60 * 1000 + defaultTTL = 60 * 60 * 1000 +) + +var idPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$`) +var hostPattern = regexp.MustCompile(`^[a-zA-Z0-9](?:[a-zA-Z0-9.-]*[a-zA-Z0-9])?$`) + +func validateID(id string) error { + if !idPattern.MatchString(id) { + return errf("unknown") + } + return nil +} + +func validateParams(principal string, p Params) error { + if err := validateID(principal); err != nil { + return err + } + if p.Name != EventName { + return errf("unknown") + } + if err := validateID(p.Arguments.QueueID); err != nil || p.Arguments.QueueID != principal { + return errf("unknown") + } + if p.Delivery.Mode != "webhook" { + return errf("unknown") + } + if err := validateSecret(p.Delivery.Secret); err != nil { + return err + } + if err := validateCursor(p.Cursor); err != nil { + return err + } + if _, err := parseTTL(p.TTL); err != nil { + return err + } + if err := validateCallbackURL(p.Delivery.URL); err != nil { + return err + } + if len(mustMarshal(p)) > maxParamsBytes { + return errf("unknown") + } + return nil +} + +func validateSecret(secret string) error { + if !strings.HasPrefix(secret, "whsec_") { + return errf("unknown") + } + raw, err := base64.StdEncoding.DecodeString(secret[len("whsec_"):]) + if err != nil || len(raw) < minSecretBytes || len(raw) > maxSecretBytes { + return errf("unknown") + } + return nil +} + +func decodeSecret(secret string) ([]byte, error) { + if err := validateSecret(secret); err != nil { + return nil, err + } + return base64.StdEncoding.DecodeString(secret[len("whsec_"):]) +} + +func validateCursor(raw json.RawMessage) error { + if len(raw) == 0 { + return nil + } + if !bytes.Equal(bytes.TrimSpace(raw), []byte("null")) { + return errf("unknown") + } + return nil +} + +func parseTTL(raw json.RawMessage) (int64, error) { + if len(raw) == 0 || bytes.Equal(bytes.TrimSpace(raw), []byte("null")) { + return defaultTTL, nil + } + trimmed := bytes.TrimSpace(raw) + if len(trimmed) == 0 || trimmed[0] < '0' || trimmed[0] > '9' { + return 0, errf("unknown") + } + var n json.Number + if json.Unmarshal(raw, &n) != nil { + return 0, errf("unknown") + } + if strings.Contains(n.String(), ".") { + return 0, errf("unknown") + } + v, err := n.Int64() + if err != nil || v < minTTL || v > maxTTL { + return 0, errf("unknown") + } + return v, nil +} + +func validateCallbackURL(raw string) error { + if len(raw) > maxCallbackURLBytes || !utf8.ValidString(raw) || strings.ContainsAny(raw, "\\\\#") { + return errf("unknown") + } + u, err := url.Parse(raw) + if err != nil || u.Scheme != "https" || u.User != nil || u.Fragment != "" || u.Opaque != "" || u.Host == "" { + return errf("unknown") + } + host := strings.ToLower(strings.TrimSuffix(u.Hostname(), ".")) + if host == "localhost" || strings.HasSuffix(host, ".localhost") || strings.HasSuffix(host, ".local") { + return errf("unknown") + } + if ip, err := netip.ParseAddr(host); err == nil { + if !publicIP(ip) { + return errf("unknown") + } + return nil + } + if !hostPattern.MatchString(host) || !strings.Contains(host, ".") || strings.Contains(host, "..") { + return errf("unknown") + } + for _, label := range strings.Split(host, ".") { + if len(label) > 63 || strings.HasPrefix(label, "-") || strings.HasSuffix(label, "-") { + return errf("unknown") + } + } + return nil +} + +func publicIP(ip netip.Addr) bool { + ip = ip.Unmap() + if !ip.IsValid() || ip.Zone() != "" || !ip.IsGlobalUnicast() || ip.IsPrivate() || ip.IsLoopback() || ip.IsLinkLocalUnicast() || ip.IsMulticast() || ip.IsUnspecified() { + return false + } + if ip.Is4() { + b := ip.As4() + if b[0] == 0 || (b[0] == 192 && b[1] == 0 && b[2] == 0) || (b[0] == 192 && b[1] == 88 && b[2] == 99) || (b[0] == 198 && b[1] == 19) { + return false + } + if b[0] == 100 && b[1] >= 64 && b[1] <= 127 { + return false + } + if b[0] == 192 && b[1] == 0 && b[2] == 2 { + return false + } + if b[0] == 198 && (b[1] == 51 || b[1] == 18) { + return false + } + if b[0] == 203 && b[1] == 0 && b[2] == 113 { + return false + } + if b[0] >= 240 { + return false + } + return true + } + if ip.Is6() { + b := ip.As16() + // Only globally allocated unicast IPv6; exclude special-purpose, + // transition mechanisms, benchmarking and documentation allocations. + if b[0]&0xe0 != 0x20 || (b[0] == 0x20 && b[1] == 0x01 && b[2] < 2) || (b[0] == 0x20 && b[1] == 0x02) || (b[0] == 0x3f && b[1] == 0xff) { + return false + } + if b[0] == 0x64 && b[1] == 0xff { + return false + } + if b[0] == 0x20 && b[1] == 0x01 && b[2] == 0x0d && b[3] == 0xb8 { + return false + } + if b[0] == 0xfd { + return false + } + if b[0] == 0xfe && b[1] == 0x80 { + return false + } + } + return true +} + +func publicAddrs(addrs []net.IPAddr) bool { + if len(addrs) == 0 { + return false + } + for _, addr := range addrs { + ip, ok := netip.AddrFromSlice(addr.IP) + if !ok || addr.Zone != "" || !publicIP(ip) { + return false + } + } + return true +} + +func mustMarshal(v any) []byte { + b, _ := json.Marshal(v) + return b +} diff --git a/internal/dotevents/wire_test.go b/internal/dotevents/wire_test.go new file mode 100644 index 0000000..49d034a --- /dev/null +++ b/internal/dotevents/wire_test.go @@ -0,0 +1,47 @@ +package dotevents + +import ( + "context" + "encoding/json" + "io" + "net/http" + "testing" +) + +func TestOfficialWireOccurrence(t *testing.T) { + var occurrence map[string]json.RawMessage + m, _, close, _ := withFixtureServerURL(t, func(w http.ResponseWriter, r *http.Request) { + b, _ := io.ReadAll(r.Body) + var v map[string]json.RawMessage + _ = json.Unmarshal(b, &v) + if string(v["type"]) == `"verification"` { + var c string + _ = json.Unmarshal(v["challenge"], &c) + _ = json.NewEncoder(w).Encode(map[string]string{"challenge": c}) + return + } + occurrence = v + w.WriteHeader(204) + }) + defer close() + if len(m.List("alice")) != 1 { + t.Error("event discovery unavailable before subscription") + } + if _, e := m.Subscribe(context.Background(), "alice", testParams("alice")); e != nil { + t.Fatal(e) + } + if e := m.Deliver(context.Background(), "alice", "alice", Notice{"evt.official", "alice", "job.official", "answer"}); e != nil { + t.Fatal(e) + } + for _, k := range []string{"eventId", "name", "timestamp", "data", "cursor"} { + if _, ok := occurrence[k]; !ok { + t.Errorf("missing official occurrence field %s", k) + } + } + if _, ok := occurrence["type"]; ok { + t.Error("occurrence used control type") + } + if string(occurrence["cursor"]) != "null" { + t.Error("cursor must be null") + } +} From 8f01eed41594b168ec697b23229423cb7c6c6724 Mon Sep 17 00:00:00 2001 From: Pyuyi <204892921+VeraPyuyi@users.noreply.github.com> Date: Mon, 19 Oct 2026 11:49:21 +0800 Subject: [PATCH 07/47] fix(dotevents): align occurrence identity and enforce private store boundary Use source event ID in both webhook-id and eventId; preserve caller occurrence time across delayed and reopened retries. Start new stores disabled. Cache verification for five minutes by authenticated principal and URL, invalidate on revocation, and preserve accepted subscriptions without TTL extension on generation-safe refresh failure. Validate canonical local non-root paths and every ancestor on every transaction; reject Windows device/UNC/ADS/reserved paths and linked ancestors; protect directory and owned files on reads and writes; validate bounded marker contents. Add independent HTTP identity/timestamp assertions, complete-fixture strict mutation tests with zero-I/O counters, cache/revocation tests, Windows junction/DACL and Unix permission/link tests. RED: go test ./internal/dotevents -run TestReview -count=1 failed on header/body identity, refresh challenge count, default-enabled new store, invalid marker and noncanonical paths. GREEN: focused tests PASS 2.582s; race PASS 4.788s with CGO/MinGW; vet PASS; Linux amd64 test binary cross-compile PASS (not executed); git diff --check PASS. No live external network, new dependencies, or runtime integration. --- docs/DOT_EVENTS.md | 10 +- internal/dotevents/cache_revision_test.go | 133 ++++++++++++++++++ internal/dotevents/delivery.go | 4 +- internal/dotevents/events_test.go | 13 +- internal/dotevents/identity.go | 13 -- internal/dotevents/lifecycle_test.go | 8 +- internal/dotevents/manager.go | 48 ++++++- internal/dotevents/multiple_test.go | 4 +- internal/dotevents/paths.go | 88 ++++++++++++ internal/dotevents/paths_unix_test.go | 46 ++++++ internal/dotevents/paths_windows_test.go | 72 ++++++++++ internal/dotevents/revision_test.go | 96 +++++++++++++ internal/dotevents/state.go | 3 +- internal/dotevents/store.go | 44 +++--- internal/dotevents/testutil_test.go | 6 + internal/dotevents/types.go | 9 +- .../dotevents/validation_revision_test.go | 85 +++++++++++ internal/dotevents/wire_test.go | 3 +- 18 files changed, 621 insertions(+), 64 deletions(-) create mode 100644 internal/dotevents/cache_revision_test.go delete mode 100644 internal/dotevents/identity.go create mode 100644 internal/dotevents/paths.go create mode 100644 internal/dotevents/paths_unix_test.go create mode 100644 internal/dotevents/paths_windows_test.go create mode 100644 internal/dotevents/revision_test.go create mode 100644 internal/dotevents/validation_revision_test.go diff --git a/docs/DOT_EVENTS.md b/docs/DOT_EVENTS.md index da7c9e4..c8871aa 100644 --- a/docs/DOT_EVENTS.md +++ b/docs/DOT_EVENTS.md @@ -2,21 +2,21 @@ `internal/dotevents` implements the subscription store and outbound webhook transport for MCP Events (2026-07-28), following the saved routing design and OpenAI's [MCP Events guide](https://developers.openai.com/plugins/build/mcp-events). Integration into MCP discovery and the runtime worker is a separate task. This package does not implement poll, stream, gap recovery, queue execution, or model readiness. -`Open(absoluteDir, key, delivery)` requires a caller-supplied 32-byte key. A nil delivery or zero-value `Delivery` uses public HTTPS with normal certificate verification. The package has no production switch to permit private callbacks. Tests inject an unexported HTTP client with explicit local fixture routing and trusted fixture certificates. +`Open(absoluteDir, key, delivery)` requires a caller-supplied 32-byte key. New stores start disabled. The authenticated owner must explicitly authorize activation through `SetEnabled(true)`; initialization never assumes a prior disable call. Existing stores retain their explicit activation state. A nil delivery or zero-value `Delivery` uses public HTTPS with normal certificate verification. The package has no production switch to permit private callbacks. Tests inject an unexported HTTP client with explicit local fixture routing and trusted fixture certificates. The authenticated adapter supplies `principal`; the caller's `arguments.queue_id` must equal that principal. `List` advertises `dot.task.available` before a subscription exists. `Params` accepts `name`, `arguments:{queue_id}`, `delivery:{mode:"webhook",url,secret}`, optional null `cursor`, and raw `ttlMs`. Its JSON decoder rejects unknown/duplicate fields and null required objects. The adapter must enforce the request byte limit before decoding too. Omitted or null TTL means one hour, with an allowed range of one minute to one day. No non-expiring grant is supported. Secrets use `whsec_` plus standard base64 encoding of 24–64 bytes. URLs are limited to 2048 bytes and params to 16 KiB. -Subscription identity is SHA-256 over the principal, event, arguments, mode, and exact callback URL, excluding the secret and TTL. Refresh keeps that identity. The encrypted state is persisted before callback verification. Every refresh gets a random generation; publishing the verification result requires the same generation after network I/O. Pause invalidates all callback verifications and generations, including across disable/re-enable. Unsubscribe selects the same identity without requiring a secret and is idempotent. The integration adapter must map the official unsubscribe request into this internal selection struct. +Subscription identity is SHA-256 over the principal, event, arguments, mode, and exact callback URL, excluding the secret and TTL. Refresh keeps that identity. The encrypted state is persisted before callback verification. Successful callback verification is cached by authenticated principal and exact callback URL for five minutes, bounded by subscription expiry. A cache hit refreshes without challenge I/O. Pause invalidates verification and cache; unsubscribe invalidates cache for that principal/URL. Every refresh gets a random generation; publishing the verification result requires the same generation after network I/O. After cache expiry, a failed challenge restores the previous accepted subscription only when the generation still matches, without extending its TTL or cache; it never restores a revoked subscription. Pause invalidates all callback verifications and generations, including across disable/re-enable. Unsubscribe selects the same identity without requiring a secret and is idempotent. The integration adapter must map the official unsubscribe request into this internal selection struct. -Verification posts signed `{ "type":"verification", "challenge":"" }`. The unique nonce is used for one attempt bounded to ten seconds, below the five-minute verification maximum. A successful response must be 2xx with bounded JSON `{ "challenge":"" }`; comparison uses constant time. An event occurrence contains `eventId`, `name`, an RFC3339 timestamp, `data:{queue_id,job_id,mode}`, and `cursor:null`. It never includes task prompts. Occurrence bytes are persisted on the first attempt and reused for retries. +Verification posts signed `{ "type":"verification", "challenge":"" }`. The unique nonce is used for one attempt bounded to ten seconds, below the five-minute verification maximum. A successful response must be 2xx with bounded JSON `{ "challenge":"" }`; comparison uses constant time. An event occurrence contains `eventId`, `name`, the caller-supplied `Notice.OccurredAt` serialized as RFC3339Nano (including its offset and fractional precision), `data:{queue_id,job_id,mode}`, and `cursor:null`. It never includes task prompts. Occurrence bytes are persisted on the first attempt and reused for retries. -Standard Webhooks signs the exact UTF-8 bytes with HMAC-SHA256 over `webhook-id.webhook-timestamp.body`. `webhook-timestamp` uses Unix seconds. The `webhook-id` is derived from subscription ID plus notice event ID, so each subscription has a distinct stable delivery identity. Each POST includes `X-MCP-Subscription-Id`. Refreshing a secret preserves the previous secret for dual signatures during a 60-second grace window. Expired subscriptions do not deliver. +Standard Webhooks signs the exact UTF-8 bytes with HMAC-SHA256 over `webhook-id.webhook-timestamp.body`. `webhook-timestamp` uses Unix seconds. The `webhook-id` equals the body's `eventId`, both taken unchanged from `Notice.EventID`. Subscription-plus-event keys remain internal to the delivery ledger. Each POST includes `X-MCP-Subscription-Id`. Refreshing a secret preserves the previous secret for dual signatures during a 60-second grace window. Expired subscriptions do not deliver. Every `Deliver` visits all matching verified subscriptions in sorted identity order, making at most one HTTP attempt per subscription. It reserves attempts atomically under the cross-process state lock before I/O. Parallel workers receive `in_flight` with a retry delay while a reservation is active; a crashed worker consumes an attempt and its reservation expires after eleven seconds. A 2xx records receipt only. 410/413 and other 4xx are terminal, except 429. Network failures, 5xx, and 429 permit three retries after 1s, 2s, and 4s, for four total attempts. No sleeps or background worker exist here: the caller/outbox schedules `Error.RetryAfter`. An early call returns the remaining delay without sending. Terminal and acknowledged records suppress further sends. There is no exactly-once guarantee after an ambiguous callback outcome. Storage capacity is 100 subscriptions and 1000 delivery records; reaching the delivery bound fails closed and requires a future explicit retention policy. Outbound transport disables environment proxies, redirects, connection reuse, and implicit DNS during socket dialing. Every new connection resolves with context, rejects the entire answer set if any address is non-public, then dials validated literal IPs while HTTP retains the original hostname for TLS SNI and certificate checks. Loopback, private, link-local, CGNAT, documentation, reserved, multicast, unspecified, IPv6 transition, and special-purpose ranges are rejected. Query strings are allowed and never copied into errors. Timeouts are at most ten seconds; response headers and verification bodies are limited to 16 KiB; event response bodies are discarded without reading; event bodies are capped at 256 KiB. -State uses AES-256-GCM, private directory/file permissions (protected current-user DACL on Windows), regular-file and hardlink checks, a five-second bounded flock, synced temporary files, and atomic replacement. Separate files are `events.state.de`, `dot_events.lock`, and `dot_events.initialized`. Missing state after initialization and invalid ciphertext fail closed. No queue package private functions or files are used. +State uses AES-256-GCM, private directory/file permissions (protected current-user DACL on Windows), regular-file and hardlink checks, a five-second bounded flock, synced temporary files, and atomic replacement. Separate files are `events.state.de`, `dot_events.lock`, and `dot_events.initialized`. Every transaction validates the canonical local non-root path and each ancestor, rejects ancestor symlinks/reparse points, and restores directory and owned-file privacy before reading. Windows UNC/device/alternate-stream/reserved-name paths are rejected. Marker content must be exactly `1\n` and is read with a three-byte bound. Missing state after initialization and invalid ciphertext fail closed. This uses the repository's private-directory threat model; it does not claim to defeat an actively racing process with the same user privileges. No queue package private functions or files are used. Errors are stable `dot_events_` values with optional retry duration and no callback URL, secret, response page, or filesystem path. `Status` distinguishes subscription and callback verification; `DotReady` is always false. The runtime must establish real readiness separately through authenticated claim/completion of a probe, and propagate pairing authorization changes into `SetEnabled` before allowing operations. diff --git a/internal/dotevents/cache_revision_test.go b/internal/dotevents/cache_revision_test.go new file mode 100644 index 0000000..9004d05 --- /dev/null +++ b/internal/dotevents/cache_revision_test.go @@ -0,0 +1,133 @@ +package dotevents + +import ( + "context" + "encoding/json" + "io" + "net/http" + "sync/atomic" + "testing" + "time" +) + +func TestCacheExpiryFailureAndRevocation(t *testing.T) { + var challenges atomic.Int32 + var fail atomic.Bool + m, dir, stop, _ := withFixtureServerURL(t, func(w http.ResponseWriter, r *http.Request) { + challenges.Add(1) + if fail.Load() { + w.WriteHeader(503) + return + } + echoVerification(w, r) + }) + defer stop() + p := testParams("alice") + first, e := m.Subscribe(context.Background(), "alice", p) + if e != nil { + t.Fatal(e) + } + reopened, e := Open(dir, testKey, m.delivery) + if e != nil { + t.Fatal(e) + } + if _, e = reopened.Subscribe(context.Background(), "alice", p); e != nil { + t.Fatal(e) + } + if challenges.Load() != 1 { + t.Fatal("cache not persisted") + } + _ = m.store.withLock(false, true, func(st *persistenceState) error { + for _, s := range st.Subscriptions { + s.VerifiedUntil = time.Now().Add(-time.Second) + } + return nil + }) + fail.Store(true) + if _, e = m.Subscribe(context.Background(), "alice", p); e == nil { + t.Fatal("expired cache bypassed challenge") + } + status, _ := m.Status("alice", "alice") + if !status.CallbackVerified { + t.Fatal("failed refresh destroyed accepted subscription") + } + _ = m.store.withLock(false, false, func(st *persistenceState) error { + if st.Subscriptions[first.ID].ExpiresAt.After(first.ExpiresAt.Add(time.Second)) { + t.Error("failed refresh extended TTL") + } + return nil + }) + if e = m.SetEnabled(false); e != nil { + t.Fatal(e) + } + if e = m.SetEnabled(true); e != nil { + t.Fatal(e) + } + if _, e = m.Subscribe(context.Background(), "alice", p); e == nil { + t.Fatal("pause retained cache") + } + status, _ = m.Status("alice", "alice") + if status.CallbackVerified { + t.Fatal("failed post-revoke refresh revived callback") + } + fail.Store(false) + if _, e = m.Subscribe(context.Background(), "alice", p); e != nil { + t.Fatal(e) + } + if e = m.Unsubscribe(context.Background(), "alice", p); e != nil { + t.Fatal(e) + } + before := challenges.Load() + if _, e = m.Subscribe(context.Background(), "alice", p); e != nil { + t.Fatal(e) + } + if challenges.Load() != before+1 { + t.Fatal("unsubscribe retained cache") + } +} +func TestFixedOccurrenceDelayedReopened(t *testing.T) { + original := time.Date(2020, 2, 3, 4, 5, 6, 123456789, time.FixedZone("offset", 3600)) + var sends atomic.Int32 + m, dir, stop, _ := withFixtureServerURL(t, func(w http.ResponseWriter, r *http.Request) { + b, _ := io.ReadAll(r.Body) + var v map[string]json.RawMessage + _ = json.Unmarshal(b, &v) + if string(v["type"]) == `"verification"` { + var c string + _ = json.Unmarshal(v["challenge"], &c) + _ = json.NewEncoder(w).Encode(map[string]string{"challenge": c}) + return + } + var stamp string + _ = json.Unmarshal(v["timestamp"], &stamp) + if stamp != original.Format(time.RFC3339Nano) { + t.Errorf("occurrence timestamp %s", stamp) + } + if r.Header.Get("webhook-id") != "evt.delayed" { + t.Error("source identity changed") + } + sends.Add(1) + w.WriteHeader(500) + }) + defer stop() + if _, e := m.Subscribe(context.Background(), "alice", testParams("alice")); e != nil { + t.Fatal(e) + } + reopened, e := Open(dir, testKey, m.delivery) + if e != nil { + t.Fatal(e) + } + n := Notice{EventID: "evt.delayed", QueueID: "alice", JobID: "job.delayed", Mode: "answer", OccurredAt: original} + _ = reopened.Deliver(context.Background(), "alice", "alice", n) + _ = m.store.withLock(false, true, func(st *persistenceState) error { + for _, r := range st.Deliveries { + r.NextAttempt = time.Time{} + } + return nil + }) + n.OccurredAt = time.Now() + _ = m.Deliver(context.Background(), "alice", "alice", n) + if sends.Load() != 2 { + t.Fatal("missing attempts") + } +} diff --git a/internal/dotevents/delivery.go b/internal/dotevents/delivery.go index 39d5c00..4ade029 100644 --- a/internal/dotevents/delivery.go +++ b/internal/dotevents/delivery.go @@ -132,7 +132,7 @@ func (d *Delivery) verify(ctx context.Context, sub *subscriptionRecord) error { return nil } func (d *Delivery) deliverBytes(ctx context.Context, sub *subscriptionRecord, id string, body []byte) (string, error) { - resp, e := d.post(ctx, sub, deliveryID(sub.ID, id), body) + resp, e := d.post(ctx, sub, id, body) if e != nil { return "network", errf("network") } @@ -164,7 +164,7 @@ func eventBodyBytes(n Notice) []byte { Mode string `json:"mode"` } `json:"data"` Cursor any `json:"cursor"` - }{EventID: n.EventID, Name: EventName, Timestamp: time.Now().UTC(), Data: struct { + }{EventID: n.EventID, Name: EventName, Timestamp: n.OccurredAt, Data: struct { QueueID string `json:"queue_id"` JobID string `json:"job_id"` Mode string `json:"mode"` diff --git a/internal/dotevents/events_test.go b/internal/dotevents/events_test.go index d011dce..e9a8458 100644 --- a/internal/dotevents/events_test.go +++ b/internal/dotevents/events_test.go @@ -12,6 +12,7 @@ import ( "strings" "sync" "testing" + "time" ) func TestInvalidSubscriptions(t *testing.T) { @@ -34,7 +35,7 @@ func TestInvalidSubscriptions(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { mgr, _ := openTestManager(t) - p := testParams(tt.principal) + p := testParams("alice") tt.mutate(&p) if _, err := mgr.Subscribe(context.Background(), tt.principal, p); err == nil { t.Fatal("accepted invalid subscription") @@ -82,7 +83,7 @@ func TestSubscribeVerifyDeliverAndPersist(t *testing.T) { if err != nil || !status.Subscribed || !status.CallbackVerified || status.DotReady { t.Fatalf("unexpected status %#v", status) } - notice := Notice{EventID: "evt.1", QueueID: "alice", JobID: "job.1", Mode: "answer"} + notice := Notice{OccurredAt: time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC), EventID: "evt.1", QueueID: "alice", JobID: "job.1", Mode: "answer"} if err := mgr.Deliver(context.Background(), "alice", "alice", notice); err != nil { t.Fatal(err) } @@ -134,7 +135,7 @@ func TestDuplicateEventAck(t *testing.T) { if _, err := mgr.Subscribe(context.Background(), "alice", testParams("alice")); err != nil { t.Fatal(err) } - notice := Notice{EventID: "evt.dup", QueueID: "alice", JobID: "job.1", Mode: "task"} + notice := Notice{OccurredAt: time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC), EventID: "evt.dup", QueueID: "alice", JobID: "job.1", Mode: "task"} if err := mgr.Deliver(context.Background(), "alice", "alice", notice); err != nil { t.Fatal(err) } @@ -170,7 +171,7 @@ func TestSecretRotationGrace(t *testing.T) { if _, err := mgr.Subscribe(context.Background(), "alice", second); err != nil { t.Fatal(err) } - notice := Notice{EventID: "evt.rotate", QueueID: "alice", JobID: "job.2", Mode: "answer"} + notice := Notice{OccurredAt: time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC), EventID: "evt.rotate", QueueID: "alice", JobID: "job.2", Mode: "answer"} if err := mgr.Deliver(context.Background(), "alice", "alice", notice); err != nil { t.Fatal(err) } @@ -196,7 +197,7 @@ func TestDisabledBlocksDelivery(t *testing.T) { if err := mgr.SetEnabled(false); err != nil { t.Fatal(err) } - if err := mgr.Deliver(context.Background(), "alice", "alice", Notice{EventID: "evt.off", QueueID: "alice", JobID: "job.1", Mode: "answer"}); asEventsError(err).Code != "disabled" { + if err := mgr.Deliver(context.Background(), "alice", "alice", Notice{OccurredAt: time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC), EventID: "evt.off", QueueID: "alice", JobID: "job.1", Mode: "answer"}); asEventsError(err).Code != "disabled" { t.Fatalf("expected disabled, got %v", err) } } @@ -233,7 +234,7 @@ func TestConcurrentSubscribeAndDeliver(t *testing.T) { }() } wg.Wait() - notice := Notice{EventID: "evt.race", QueueID: "alice", JobID: "job.race", Mode: "answer"} + notice := Notice{OccurredAt: time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC), EventID: "evt.race", QueueID: "alice", JobID: "job.race", Mode: "answer"} for i := 0; i < 8; i++ { wg.Add(1) go func() { diff --git a/internal/dotevents/identity.go b/internal/dotevents/identity.go deleted file mode 100644 index 69cac14..0000000 --- a/internal/dotevents/identity.go +++ /dev/null @@ -1,13 +0,0 @@ -package dotevents - -import ( - "crypto/sha256" - "encoding/hex" -) - -// The occurrence eventId identifies the source notice. Standard Webhooks IDs -// identify a delivery to one subscription and remain stable across retries. -func deliveryID(subscriptionID, eventID string) string { - sum := sha256.Sum256([]byte(deliveryKey(subscriptionID, eventID))) - return "evt_" + hex.EncodeToString(sum[:]) -} diff --git a/internal/dotevents/lifecycle_test.go b/internal/dotevents/lifecycle_test.go index 652ef7f..ce30dee 100644 --- a/internal/dotevents/lifecycle_test.go +++ b/internal/dotevents/lifecycle_test.go @@ -103,7 +103,7 @@ func TestRetriesPersistOneAttemptAndTerminal(t *testing.T) { if _, e := m.Subscribe(context.Background(), "alice", testParams("alice")); e != nil { t.Fatal(e) } - n := Notice{"evt.retry", "alice", "job.retry", "answer"} + n := Notice{EventID: "evt.retry", QueueID: "alice", JobID: "job.retry", Mode: "answer", OccurredAt: time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)} e := m.Deliver(context.Background(), "alice", "alice", n) if e == nil || sends.Load() != 1 { t.Fatalf("attempt %v %d", e, sends.Load()) @@ -142,7 +142,7 @@ func TestRetriesPersistOneAttemptAndTerminal(t *testing.T) { t.Fatalf("unbounded sends %d", sends.Load()) } for i := range ids { - if ids[i] != deliveryID(subscriptionID("alice", testParams("alice")), n.EventID) || !bytes.Equal(bodies[0], bodies[i]) { + if ids[i] != n.EventID || !bytes.Equal(bodies[0], bodies[i]) { t.Fatal("retry changed identity or body") } } @@ -181,7 +181,7 @@ func TestUnsubscribeExpiryEncryptionAndSignatures(t *testing.T) { if time.Until(sub.ExpiresAt) > time.Hour { t.Fatal("unbounded null ttl") } - if e = m.Deliver(context.Background(), "alice", "alice", Notice{"evt.signed", "alice", "job.signed", "task"}); e != nil { + if e = m.Deliver(context.Background(), "alice", "alice", Notice{EventID: "evt.signed", QueueID: "alice", JobID: "job.signed", Mode: "task", OccurredAt: time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)}); e != nil { t.Fatal(e) } if !checked.Load() { @@ -197,7 +197,7 @@ func TestUnsubscribeExpiryEncryptionAndSignatures(t *testing.T) { } return nil }) - if e = m.Deliver(context.Background(), "alice", "alice", Notice{"evt.expired", "alice", "job.signed", "task"}); e == nil { + if e = m.Deliver(context.Background(), "alice", "alice", Notice{EventID: "evt.expired", QueueID: "alice", JobID: "job.signed", Mode: "task", OccurredAt: time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)}); e == nil { t.Fatal("expired sent") } p.Delivery.Secret = "" diff --git a/internal/dotevents/manager.go b/internal/dotevents/manager.go index 561bb24..f4a3a59 100644 --- a/internal/dotevents/manager.go +++ b/internal/dotevents/manager.go @@ -31,6 +31,7 @@ func (m *Manager) SetEnabled(enabled bool) error { if !enabled { for _, s := range st.Subscriptions { s.CallbackVerified = false + s.VerifiedUntil = time.Time{} s.Generation = "" } } @@ -48,6 +49,8 @@ func (m *Manager) Subscribe(ctx context.Context, principal string, p Params) (Su return Subscription{}, e } now := time.Now().UTC() + var previous *subscriptionRecord + var cached bool rec := &subscriptionRecord{ID: id, Generation: gen, Principal: principal, Params: p, Secret: p.Delivery.Secret, ExpiresAt: now.Add(time.Duration(ttl) * time.Millisecond), CreatedAt: now, UpdatedAt: now} if e = m.store.withLock(false, true, func(st *persistenceState) error { if !st.Enabled { @@ -59,6 +62,20 @@ func (m *Manager) Subscribe(ctx context.Context, principal string, p Params) (Su } } old := st.Subscriptions[id] + if old != nil { + cp := *old + previous = &cp + } + // Cache is scoped to authenticated principal plus exact URL and bounded + // by both verification lifetime and subscription expiry. + for _, s := range st.Subscriptions { + if s.Principal == principal && s.Params.Delivery.URL == p.Delivery.URL && s.CallbackVerified && s.VerifiedUntil.After(now) && s.ExpiresAt.After(now) { + cached = true + rec.CallbackVerified = true + rec.VerifiedUntil = s.VerifiedUntil + break + } + } if old == nil && len(st.Subscriptions) >= maxActiveSubs { return errf("capacity") } @@ -77,7 +94,25 @@ func (m *Manager) Subscribe(ctx context.Context, principal string, p Params) (Su }); e != nil { return Subscription{}, e } + if cached { + return Subscription{ID: id, ExpiresAt: rec.ExpiresAt, CallbackVerified: true}, nil + } if e = m.delivery.verify(ctx, rec); e != nil { + // Restore a previously accepted refresh only if no revoke/refresh won + // the generation race. Never extend its TTL or cache after failure. + restoreErr := m.store.withLock(false, true, func(st *persistenceState) error { + current := st.Subscriptions[id] + if !st.Enabled || current == nil || current.Generation != gen { + return errf("stale") + } + if previous != nil && previous.CallbackVerified && previous.ExpiresAt.After(time.Now()) { + st.Subscriptions[id] = previous + } + return nil + }) + if restoreErr != nil { + return Subscription{}, restoreErr + } return Subscription{}, e } e = m.store.withLock(false, true, func(st *persistenceState) error { @@ -89,6 +124,7 @@ func (m *Manager) Subscribe(ctx context.Context, principal string, p Params) (Su return errf("stale") } s.CallbackVerified = true + s.VerifiedUntil = time.Now().Add(5 * time.Minute) return nil }) if e != nil { @@ -103,7 +139,15 @@ func (m *Manager) Unsubscribe(ctx context.Context, principal string, p Params) e if validateID(principal) != nil || p.Arguments.QueueID != principal || p.Name != EventName || p.Delivery.Mode != "webhook" || validateCallbackURL(p.Delivery.URL) != nil { return errf("unknown") } - return m.store.withLock(false, true, func(st *persistenceState) error { delete(st.Subscriptions, subscriptionID(principal, p)); return nil }) + return m.store.withLock(false, true, func(st *persistenceState) error { + delete(st.Subscriptions, subscriptionID(principal, p)) + for _, s := range st.Subscriptions { + if s.Principal == principal && s.Params.Delivery.URL == p.Delivery.URL { + s.VerifiedUntil = time.Time{} + } + } + return nil + }) } func (m *Manager) List(principal string) []Definition { if validateID(principal) != nil { @@ -112,7 +156,7 @@ func (m *Manager) List(principal string) []Definition { return []Definition{{Name: EventName, Description: "A queued task is available; retrieve it using authorized queue tools."}} } func (m *Manager) Deliver(ctx context.Context, principal, queueID string, n Notice) error { - if validateID(principal) != nil || principal != queueID || n.QueueID != queueID || validateID(n.EventID) != nil || validateID(n.JobID) != nil || (n.Mode != "answer" && n.Mode != "task") { + if n.OccurredAt.IsZero() || n.OccurredAt.Year() < 1 || n.OccurredAt.Year() > 9999 || validateID(principal) != nil || principal != queueID || n.QueueID != queueID || validateID(n.EventID) != nil || validateID(n.JobID) != nil || (n.Mode != "answer" && n.Mode != "task") { return errf("unknown") } var ids []string diff --git a/internal/dotevents/multiple_test.go b/internal/dotevents/multiple_test.go index aa38d7b..7c9182a 100644 --- a/internal/dotevents/multiple_test.go +++ b/internal/dotevents/multiple_test.go @@ -42,7 +42,7 @@ func TestMultipleSubscriptionsAndRevokeDuringDelivery(t *testing.T) { if _, e := m.Subscribe(context.Background(), "alice", second); e != nil { t.Fatal(e) } - if e := m.Deliver(context.Background(), "alice", "alice", Notice{"evt.multi", "alice", "job.multi", "answer"}); e != nil { + if e := m.Deliver(context.Background(), "alice", "alice", Notice{EventID: "evt.multi", QueueID: "alice", JobID: "job.multi", Mode: "answer", OccurredAt: time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)}); e != nil { t.Fatal(e) } if sends.Load() != 2 { @@ -54,7 +54,7 @@ func TestMultipleSubscriptionsAndRevokeDuringDelivery(t *testing.T) { block.Store(true) done := make(chan error, 1) go func() { - done <- m.Deliver(context.Background(), "alice", "alice", Notice{"evt.block", "alice", "job.multi", "task"}) + done <- m.Deliver(context.Background(), "alice", "alice", Notice{EventID: "evt.block", QueueID: "alice", JobID: "job.multi", Mode: "task", OccurredAt: time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)}) }() <-entered if e := m.SetEnabled(false); e != nil { diff --git a/internal/dotevents/paths.go b/internal/dotevents/paths.go new file mode 100644 index 0000000..ebc71e9 --- /dev/null +++ b/internal/dotevents/paths.go @@ -0,0 +1,88 @@ +package dotevents + +import ( + "os" + "path/filepath" + "runtime" + "strings" + "unicode" +) + +func validateStorePath(dir string) error { + if !filepath.IsAbs(dir) || filepath.Clean(dir) != dir || len(dir) > 32768 || strings.ContainsFunc(dir, unicode.IsControl) { + return errf("unknown") + } + vol := filepath.VolumeName(dir) + if dir == vol+string(os.PathSeparator) { + return errf("unknown") + } + if runtime.GOOS == "windows" { + if len(vol) != 2 || vol[1] != ':' || strings.Contains(dir[2:], ":") || strings.Contains(dir, "/") { + return errf("unknown") + } + for _, part := range strings.Split(dir[3:], `\`) { + if part == "" || strings.HasSuffix(part, ".") || strings.HasSuffix(part, " ") { + return errf("unknown") + } + name := strings.ToUpper(strings.SplitN(part, ".", 2)[0]) + if name == "CON" || name == "PRN" || name == "AUX" || name == "NUL" || (len(name) == 4 && (strings.HasPrefix(name, "COM") || strings.HasPrefix(name, "LPT")) && name[3] >= '1' && name[3] <= '9') { + return errf("unknown") + } + } + } + return nil +} +func checkAncestors(dir string, create bool) error { + if e := validateStorePath(dir); e != nil { + return e + } + var chain []string + for p := dir; ; p = filepath.Dir(p) { + chain = append(chain, p) + if filepath.Dir(p) == p { + break + } + } + for i := len(chain) - 1; i >= 0; i-- { + p := chain[i] + info, e := os.Lstat(p) + if os.IsNotExist(e) && create { + if e = os.Mkdir(p, 0700); e != nil { + return errf("unknown") + } + info, e = os.Lstat(p) + } + if e != nil || !info.IsDir() || rejectLink(info) != nil { + return errf("unknown") + } + } + return privatePath(dir, true) +} + +// Recheck and protect owned files on reads as well as writes. The enclosing +// directory is private; this follows the repository's same-user threat model. +func protectFile(path string, required bool) error { + info, e := os.Lstat(path) + if os.IsNotExist(e) && !required { + return nil + } + if e != nil || !info.Mode().IsRegular() || rejectLink(info) != nil { + return errf("corrupt") + } + f, e := os.Open(path) + if e != nil { + return errf("corrupt") + } + fi, e := f.Stat() + if e != nil || rejectHardlink(f, fi) != nil { + f.Close() + return errf("corrupt") + } + if e = f.Close(); e != nil { + return errf("corrupt") + } + if privatePath(path, false) != nil { + return errf("corrupt") + } + return nil +} diff --git a/internal/dotevents/paths_unix_test.go b/internal/dotevents/paths_unix_test.go new file mode 100644 index 0000000..7f6153b --- /dev/null +++ b/internal/dotevents/paths_unix_test.go @@ -0,0 +1,46 @@ +//go:build !windows + +package dotevents + +import ( + "os" + "path/filepath" + "testing" +) + +func TestUnixPrivacyEveryTransaction(t *testing.T) { + m, dir := openTestManager(t) + paths := []string{dir, filepath.Join(dir, stateFileName), filepath.Join(dir, markerName), filepath.Join(dir, lockName)} + for _, p := range paths { + if e := os.Chmod(p, 0777); e != nil { + t.Fatal(e) + } + } + if _, e := m.Status("alice", "alice"); e != nil { + t.Fatal(e) + } + for i, p := range paths { + fi, e := os.Stat(p) + if e != nil { + t.Fatal(e) + } + want := os.FileMode(0600) + if i == 0 { + want = 0700 + } + if fi.Mode().Perm() != want { + t.Errorf("mode %s %o", p, fi.Mode().Perm()) + } + } +} +func TestUnixLinkedAncestorOnReopen(t *testing.T) { + base := t.TempDir() + target := t.TempDir() + link := filepath.Join(base, "link") + if e := os.Symlink(target, link); e != nil { + t.Fatal(e) + } + if _, e := Open(filepath.Join(link, "events"), testKey, nil); e == nil { + t.Fatal("symlink ancestor accepted") + } +} diff --git a/internal/dotevents/paths_windows_test.go b/internal/dotevents/paths_windows_test.go new file mode 100644 index 0000000..8f162ef --- /dev/null +++ b/internal/dotevents/paths_windows_test.go @@ -0,0 +1,72 @@ +//go:build windows + +package dotevents + +import ( + "golang.org/x/sys/windows" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +func TestWindowsStoreBoundary(t *testing.T) { + for _, p := range []string{`\\server\share\events`, `\\?\C:\events`, `\\.\C:\events`, `C:\events:stream`, `C:\events\NUL`, `C:\events\trailing.`, `C:\events\trailing `} { + if _, e := Open(p, testKey, nil); e == nil { + t.Errorf("unsafe path %s", p) + } + } + base := t.TempDir() + target := t.TempDir() + link := filepath.Join(base, "junction") + out, e := exec.Command("cmd", "/c", "mklink", "/J", link, target).CombinedOutput() + if e != nil { + t.Fatalf("junction fixture: %v %s", e, out) + } + defer os.Remove(link) + if _, e = Open(filepath.Join(link, "events"), testKey, nil); e == nil { + t.Fatal("junction ancestor accepted") + } +} +func TestWindowsPrivacyEveryTransaction(t *testing.T) { + m, dir := openTestManager(t) + paths := []string{dir, filepath.Join(dir, stateFileName), filepath.Join(dir, markerName), filepath.Join(dir, lockName)} + for _, p := range paths { + sd, e := windows.SecurityDescriptorFromString("D:P(A;;FA;;;WD)") + if e != nil { + t.Fatal(e) + } + acl, _, e := sd.DACL() + if e != nil { + t.Fatal(e) + } + if e = windows.SetNamedSecurityInfo(p, windows.SE_FILE_OBJECT, windows.DACL_SECURITY_INFORMATION|windows.PROTECTED_DACL_SECURITY_INFORMATION, nil, nil, acl, nil); e != nil { + t.Fatal(e) + } + } + if _, e := m.Status("alice", "alice"); e != nil { + t.Fatal(e) + } + for _, p := range paths { + sd, e := windows.GetNamedSecurityInfo(p, windows.SE_FILE_OBJECT, windows.DACL_SECURITY_INFORMATION) + if e != nil { + t.Fatal(e) + } + acl, _, e := sd.DACL() + if e != nil { + t.Fatal(e) + } + user, e := windows.GetCurrentProcessToken().GetTokenUser() + if e != nil { + t.Fatal(e) + } + text := sd.String() + if !strings.Contains(text, user.User.Sid.String()) || strings.Contains(text, ";;;WD") { + t.Errorf("not restricted to current user: %s", text) + } + if acl.AceCount != 1 { + t.Errorf("unexpected ACL entries for %s: %d", p, acl.AceCount) + } + } +} diff --git a/internal/dotevents/revision_test.go b/internal/dotevents/revision_test.go new file mode 100644 index 0000000..41e71b6 --- /dev/null +++ b/internal/dotevents/revision_test.go @@ -0,0 +1,96 @@ +package dotevents + +import ( + "context" + "encoding/json" + "io" + "net/http" + "os" + "path/filepath" + "sync/atomic" + "testing" + "time" +) + +func TestReviewIdentityTimestampAndCache(t *testing.T) { + var challenges atomic.Int32 + var bodies [][]byte + m, dir, stop, _ := withFixtureServerURL(t, func(w http.ResponseWriter, r *http.Request) { + b, _ := io.ReadAll(r.Body) + var v map[string]json.RawMessage + _ = json.Unmarshal(b, &v) + if string(v["type"]) == `"verification"` { + challenges.Add(1) + var c string + _ = json.Unmarshal(v["challenge"], &c) + _ = json.NewEncoder(w).Encode(map[string]string{"challenge": c}) + return + } + var id string + _ = json.Unmarshal(v["eventId"], &id) + if r.Header.Get("webhook-id") != id { + t.Error("webhook-id differs from body eventId") + } + bodies = append(bodies, b) + w.WriteHeader(500) + }) + defer stop() + p := testParams("alice") + for i := 0; i < 2; i++ { + if _, e := m.Subscribe(context.Background(), "alice", p); e != nil { + t.Fatal(e) + } + } + if challenges.Load() != 1 { + t.Errorf("refresh challenges=%d", challenges.Load()) + } + n := Notice{OccurredAt: time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC), EventID: "evt.original", QueueID: "alice", JobID: "job.original", Mode: "task"} + _ = m.Deliver(context.Background(), "alice", "alice", n) + reopened, e := Open(dir, testKey, m.delivery) + if e != nil { + t.Fatal(e) + } + _ = m.store.withLock(false, true, func(s *persistenceState) error { + for _, r := range s.Deliveries { + r.NextAttempt = time.Time{} + } + return nil + }) + _ = reopened.Deliver(context.Background(), "alice", "alice", n) + if len(bodies) != 2 || string(bodies[0]) != string(bodies[1]) { + t.Fatal("reopened retry body changed") + } +} +func TestReviewNewStoreDisabledAndMarker(t *testing.T) { + m, e := Open(t.TempDir(), testKey, nil) + if e != nil { + t.Fatal(e) + } + _, e = m.Subscribe(context.Background(), "alice", testParams("alice")) + if e == nil || asEventsError(e).Code != "disabled" { + t.Errorf("new store enabled: %v", e) + } + if e = os.WriteFile(filepath.Join(m.store.dir, markerName), []byte("bad"), 0600); e != nil { + t.Fatal(e) + } + if _, e = m.Status("alice", "alice"); e == nil { + t.Error("invalid marker accepted") + } +} +func TestReviewPaths(t *testing.T) { + base := t.TempDir() + for _, p := range []string{filepath.VolumeName(base) + string(os.PathSeparator), base + string(os.PathSeparator) + ".", filepath.Join(base, "x") + string(os.PathSeparator) + ".." + string(os.PathSeparator) + "y"} { + if _, e := Open(p, testKey, nil); e == nil { + t.Errorf("unsafe path accepted %s", p) + } + } + target := t.TempDir() + link := filepath.Join(base, "link") + if e := os.Symlink(target, link); e != nil { + t.Logf("symlink unavailable: %v", e) + return + } + if _, e := Open(filepath.Join(link, "events"), testKey, nil); e == nil { + t.Fatal("linked ancestor accepted") + } +} diff --git a/internal/dotevents/state.go b/internal/dotevents/state.go index f543f6a..00c387f 100644 --- a/internal/dotevents/state.go +++ b/internal/dotevents/state.go @@ -16,6 +16,7 @@ const ( ) type subscriptionRecord struct { + VerifiedUntil time.Time `json:"verified_until"` Generation string `json:"generation"` ID string `json:"id"` Principal string `json:"principal"` @@ -60,7 +61,7 @@ type persistenceState struct { func newPersistenceState() *persistenceState { return &persistenceState{ StateVersion: stateVersion, - Enabled: true, + Enabled: false, Subscriptions: map[string]*subscriptionRecord{}, Deliveries: map[string]*deliveryRecord{}, QueueStatus: map[string]*queueStatus{}, diff --git a/internal/dotevents/store.go b/internal/dotevents/store.go index 9d9c181..7933416 100644 --- a/internal/dotevents/store.go +++ b/internal/dotevents/store.go @@ -6,9 +6,8 @@ import ( "io" "os" "path/filepath" - "strings" + "time" - "unicode" "github.com/gofrs/flock" ) @@ -22,7 +21,7 @@ func openStore(dir string, key []byte) (*store, error) { if len(key) != 32 { return nil, errf("unknown") } - if !filepath.IsAbs(dir) || len(dir) > 32768 || strings.ContainsFunc(dir, unicode.IsControl) { + if validateStorePath(dir) != nil { return nil, errf("unknown") } return &store{dir: dir, key: append([]byte(nil), key...)}, nil @@ -56,6 +55,14 @@ func (s *store) withLock(allowCreate bool, mutate bool, fn func(*persistenceStat return errf("unknown") } defer lock.Unlock() + if e := s.checkDir(false); e != nil { + return e + } + for _, name := range []string{stateFileName, markerName, lockName} { + if e := protectFile(filepath.Join(s.dir, name), false); e != nil { + return e + } + } st, err := s.loadState(allowCreate) if err != nil { @@ -90,6 +97,15 @@ func (s *store) loadState(allowCreate bool) (*persistenceState, error) { } return nil, errf("corrupt") } + markerFile, e := os.Open(marker) + if e != nil { + return nil, errf("corrupt") + } + markerData, e := io.ReadAll(io.LimitReader(markerFile, 3)) + closeErr := markerFile.Close() + if e != nil || closeErr != nil || string(markerData) != "1\n" { + return nil, errf("corrupt") + } if err := regularFile(statePath); err != nil { return nil, errf("corrupt") } @@ -191,27 +207,7 @@ func (s *store) ensureMarker() error { return nil } -func (s *store) checkDir(create bool) error { - if !filepath.IsAbs(s.dir) || len(s.dir) > 32768 || strings.ContainsFunc(s.dir, unicode.IsControl) { - return errf("unknown") - } - if create { - if os.MkdirAll(s.dir, 0700) != nil { - return errf("unknown") - } - } - info, err := os.Lstat(s.dir) - if os.IsNotExist(err) { - return errf("corrupt") - } - if err != nil || !info.IsDir() || rejectLink(info) != nil { - return errf("unknown") - } - if create && privatePath(s.dir, true) != nil { - return errf("unknown") - } - return nil -} +func (s *store) checkDir(create bool) error { return checkAncestors(s.dir, create) } func regularFile(path string) error { info, err := os.Lstat(path) diff --git a/internal/dotevents/testutil_test.go b/internal/dotevents/testutil_test.go index 548bb7b..9cb9420 100644 --- a/internal/dotevents/testutil_test.go +++ b/internal/dotevents/testutil_test.go @@ -22,6 +22,9 @@ func openTestManager(t *testing.T) (*Manager, string) { if e != nil { t.Fatal(e) } + if e = m.SetEnabled(true); e != nil { + t.Fatal(e) + } return m, dir } func testSecret() string { @@ -52,5 +55,8 @@ func withFixtureServerURL(t *testing.T, h http.HandlerFunc) (*Manager, string, f if e != nil { t.Fatal(e) } + if e = m.SetEnabled(true); e != nil { + t.Fatal(e) + } return m, dir, s.Close, fixtureCallbackURL } diff --git a/internal/dotevents/types.go b/internal/dotevents/types.go index 31c2dd4..62747a2 100644 --- a/internal/dotevents/types.go +++ b/internal/dotevents/types.go @@ -37,10 +37,11 @@ type Definition struct { } type Notice struct { - EventID string - QueueID string - JobID string - Mode string + OccurredAt time.Time + EventID string + QueueID string + JobID string + Mode string } type Status struct { diff --git a/internal/dotevents/validation_revision_test.go b/internal/dotevents/validation_revision_test.go new file mode 100644 index 0000000..6bb2d77 --- /dev/null +++ b/internal/dotevents/validation_revision_test.go @@ -0,0 +1,85 @@ +package dotevents + +import ( + "context" + "encoding/json" + "net/http" + "sync/atomic" + "testing" +) + +func TestCompleteFixtureValidationZeroIO(t *testing.T) { + var calls atomic.Int32 + m, _, stop, _ := withFixtureServerURL(t, func(w http.ResponseWriter, r *http.Request) { calls.Add(1); echoVerification(w, r) }) + defer stop() + cases := []struct { + name string + mutate func(*Params) + principal string + }{ + {"principal_queue_mismatch", func(p *Params) { p.Arguments.QueueID = "bob" }, "alice"}, + {"invalid_principal", func(*Params) {}, "bad principal"}, + {"event", func(p *Params) { p.Name = "other" }, "alice"}, + {"secret", func(p *Params) { p.Delivery.Secret = "whsec_bad" }, "alice"}, + {"ttl_string", func(p *Params) { p.TTL = json.RawMessage(`"60000"`) }, "alice"}, + {"ttl_small", func(p *Params) { p.TTL = json.RawMessage(`1`) }, "alice"}, + {"cursor", func(p *Params) { p.Cursor = json.RawMessage(`"resume"`) }, "alice"}, + {"private", func(p *Params) { p.Delivery.URL = "https://127.0.0.1/" }, "alice"}, + {"fragment", func(p *Params) { p.Delivery.URL += "#x" }, "alice"}, + {"mode", func(p *Params) { p.Delivery.Mode = "stream" }, "alice"}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + p := testParams("alice") + c.mutate(&p) + _, e := m.Subscribe(context.Background(), c.principal, p) + if e == nil || asEventsError(e).Code != "unknown" { + t.Fatalf("validation error %v", e) + } + if calls.Load() != 0 { + t.Fatal("invalid subscription caused I/O") + } + }) + } + if _, e := m.Subscribe(context.Background(), "alice", testParams("alice")); e != nil { + t.Fatal(e) + } + if calls.Load() != 1 { + t.Fatal("valid control did not reach callback") + } +} +func TestCompleteFixtureStrictMutations(t *testing.T) { + valid, _ := json.Marshal(testParams("alice")) + var base map[string]any + _ = json.Unmarshal(valid, &base) + for _, c := range []struct { + name string + mutate func(map[string]any) + }{ + {"unknown_top", func(v map[string]any) { v["extra"] = true }}, + {"unknown_arguments", func(v map[string]any) { v["arguments"].(map[string]any)["extra"] = true }}, + {"null_queue", func(v map[string]any) { v["arguments"].(map[string]any)["queue_id"] = nil }}, + {"unknown_delivery", func(v map[string]any) { v["delivery"].(map[string]any)["extra"] = true }}, + {"null_url", func(v map[string]any) { v["delivery"].(map[string]any)["url"] = nil }}, + {"case_name", func(v map[string]any) { v["Name"] = v["name"]; delete(v, "name") }}, + } { + t.Run(c.name, func(t *testing.T) { + var v map[string]any + _ = json.Unmarshal(valid, &v) + c.mutate(v) + b, _ := json.Marshal(v) + var p Params + if json.Unmarshal(b, &p) == nil { + t.Fatal("mutated complete fixture accepted") + } + }) + } + var p Params + if e := json.Unmarshal(valid, &p); e != nil { + t.Fatalf("complete control failed %v", e) + } + duplicate := append([]byte(`{"name":"other",`), valid[1:]...) + if json.Unmarshal(duplicate, &p) == nil { + t.Fatal("duplicate accepted") + } +} diff --git a/internal/dotevents/wire_test.go b/internal/dotevents/wire_test.go index 49d034a..4b8c7ef 100644 --- a/internal/dotevents/wire_test.go +++ b/internal/dotevents/wire_test.go @@ -6,6 +6,7 @@ import ( "io" "net/http" "testing" + "time" ) func TestOfficialWireOccurrence(t *testing.T) { @@ -30,7 +31,7 @@ func TestOfficialWireOccurrence(t *testing.T) { if _, e := m.Subscribe(context.Background(), "alice", testParams("alice")); e != nil { t.Fatal(e) } - if e := m.Deliver(context.Background(), "alice", "alice", Notice{"evt.official", "alice", "job.official", "answer"}); e != nil { + if e := m.Deliver(context.Background(), "alice", "alice", Notice{EventID: "evt.official", QueueID: "alice", JobID: "job.official", Mode: "answer", OccurredAt: time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)}); e != nil { t.Fatal(e) } for _, k := range []string{"eventId", "name", "timestamp", "data", "cursor"} { From accc755c0b13ce2983d3dcbb1f0d8046df5fa365 Mon Sep 17 00:00:00 2001 From: Pyuyi <204892921+VeraPyuyi@users.noreply.github.com> Date: Mon, 19 Oct 2026 12:01:43 +0800 Subject: [PATCH 08/47] feat(dotbridge): integrate queue/events runtime bridge --- cmd/dotcontrol/dot_bridge.go | 134 ++++++++++ cmd/dotcontrol/main.go | 6 + cmd/dotcontrol/runtime.go | 101 ++++++- docs/DOT_BRIDGE.md | 19 ++ internal/config/config.go | 1 + internal/dotbridge/adapters.go | 79 ++++++ internal/dotbridge/api.go | 182 +++++++++++++ internal/dotbridge/bridge_test.go | 122 +++++++++ internal/dotbridge/context.go | 16 ++ internal/dotbridge/fakevault.go | 34 +++ internal/dotbridge/mcp_test.go | 26 ++ internal/dotbridge/pairing.go | 37 +++ internal/dotbridge/probe.go | 68 +++++ internal/dotbridge/secrets.go | 46 ++++ internal/dotbridge/service.go | 427 ++++++++++++++++++++++++++++++ internal/dotbridge/testutil.go | 7 + internal/dotbridge/types.go | 105 ++++++++ internal/dotbridge/worker.go | 138 ++++++++++ internal/mcpserver/bridge.go | 159 +++++++++++ internal/mcpserver/discover.go | 76 ++++++ 20 files changed, 1782 insertions(+), 1 deletion(-) create mode 100644 cmd/dotcontrol/dot_bridge.go create mode 100644 docs/DOT_BRIDGE.md create mode 100644 internal/dotbridge/adapters.go create mode 100644 internal/dotbridge/api.go create mode 100644 internal/dotbridge/bridge_test.go create mode 100644 internal/dotbridge/context.go create mode 100644 internal/dotbridge/fakevault.go create mode 100644 internal/dotbridge/mcp_test.go create mode 100644 internal/dotbridge/pairing.go create mode 100644 internal/dotbridge/probe.go create mode 100644 internal/dotbridge/secrets.go create mode 100644 internal/dotbridge/service.go create mode 100644 internal/dotbridge/testutil.go create mode 100644 internal/dotbridge/types.go create mode 100644 internal/dotbridge/worker.go create mode 100644 internal/mcpserver/bridge.go create mode 100644 internal/mcpserver/discover.go diff --git a/cmd/dotcontrol/dot_bridge.go b/cmd/dotcontrol/dot_bridge.go new file mode 100644 index 0000000..f873cb3 --- /dev/null +++ b/cmd/dotcontrol/dot_bridge.go @@ -0,0 +1,134 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "strings" + + "github.com/dotcontrol/dotcontrol/internal/config" + "github.com/dotcontrol/dotcontrol/internal/dotbridge" + "github.com/dotcontrol/dotcontrol/internal/localapi" + "github.com/dotcontrol/dotcontrol/internal/vault" +) + +func bridgeCLI(ctx context.Context, dir string, c config.Config, v vault.Vault, args []string) error { + if len(args) == 0 { + return errors.New("bridge command required") + } + switch args[0] { + case "enable": + return bridgePost(ctx, c, v, "/admin/bridge/enable", nil) + case "disable": + return bridgePost(ctx, c, v, "/admin/bridge/disable", nil) + case "status": + return bridgePost(ctx, c, v, "/admin/bridge/status", nil) + case "pair": + return bridgePost(ctx, c, v, "/admin/bridge/pair", map[string]any{}) + case "revoke-client": + if len(args) < 2 { + return errors.New("client_id required") + } + return bridgePost(ctx, c, v, "/admin/bridge/revoke-client", map[string]string{"client_id": args[1]}) + case "probe": + if len(args) < 2 { + return errors.New("client_id required") + } + return bridgePost(ctx, c, v, "/admin/bridge/probe", map[string]string{"client_id": args[1]}) + case "jobs": + if len(args) < 2 { + return errors.New("client_id required") + } + return bridgeClientGet(ctx, dir, c.Listen, args[1], "/dot-bridge/v1/status") + case "get": + if len(args) < 3 { + return errors.New("client_id and job_id required") + } + return bridgeClientGet(ctx, dir, c.Listen, args[1], "/dot-bridge/v1/jobs/"+args[2]) + case "cancel": + if len(args) < 3 { + return errors.New("client_id and job_id required") + } + return bridgeClientPost(ctx, dir, c.Listen, args[1], "/dot-bridge/v1/jobs/"+args[2]+"/cancel", nil) + default: + return errors.New("unknown_bridge_command") + } +} + +func bridgePost(ctx context.Context, c config.Config, v vault.Vault, path string, body any) error { + token, err := v.Get("admin-token") + if err != nil { + return errors.New("local_credential_missing") + } + data, err := localapi.Client{Address: c.Listen, Token: token}.Post(ctx, path, body) + if err != nil { + return err + } + fmt.Println(string(data)) + return nil +} + +func bridgeClientGet(ctx context.Context, dir, listen, clientID, path string) error { + token, err := loadPairingToken(dir, clientID) + if err != nil { + return err + } + return bridgeClientRequest(ctx, listen, token, http.MethodGet, path, nil) +} + +func bridgeClientPost(ctx context.Context, dir, listen, clientID, path string, body any) error { + token, err := loadPairingToken(dir, clientID) + if err != nil { + return err + } + return bridgeClientRequest(ctx, listen, token, http.MethodPost, path, body) +} + +func bridgeClientRequest(ctx context.Context, listen, token, method, path string, body any) error { + var reader io.Reader + if body != nil { + b, err := json.Marshal(body) + if err != nil { + return err + } + reader = strings.NewReader(string(b)) + } + req, err := http.NewRequestWithContext(ctx, method, "http://"+listen+path, reader) + if err != nil { + return err + } + req.Header.Set("Authorization", "Bearer "+token) + if body != nil { + req.Header.Set("Content-Type", "application/json") + } + client := &http.Client{CheckRedirect: func(*http.Request, []*http.Request) error { return errors.New("redirect_refused") }} + res, err := client.Do(req) + if err != nil { + return err + } + defer res.Body.Close() + data, _ := io.ReadAll(io.LimitReader(res.Body, 1<<20)) + fmt.Println(string(data)) + if res.StatusCode >= 400 { + return errors.New("bridge_request_failed") + } + return nil +} + +func loadPairingToken(dir, clientID string) (string, error) { + path := filepath.Join(dir, "dot-bridge", "pairing-"+clientID+".json") + data, err := os.ReadFile(path) + if err != nil { + return "", errors.New("pairing_file_missing") + } + var file dotbridge.PairingFile + if json.Unmarshal(data, &file) != nil || file.Token == "" { + return "", errors.New("pairing_file_invalid") + } + return file.Token, nil +} diff --git a/cmd/dotcontrol/main.go b/cmd/dotcontrol/main.go index 9179d0d..bcc7dfd 100644 --- a/cmd/dotcontrol/main.go +++ b/cmd/dotcontrol/main.go @@ -161,6 +161,8 @@ func run(args []string) error { return errors.New("local_credential_missing") } return mcpserver.New(localapi.Client{Address: c.Listen, Token: token}).Run(ctx, &mcp.StdioTransport{}) + case "bridge": + return bridgeCLI(ctx, *dir, c, v, args[1:]) case "pair": return pair(*dir, c, v, args[1:]) case "probe": @@ -208,6 +210,7 @@ dotcontrol [--state DIRECTORY] COMMAND pause Stop future input and interrupt the current task resume Resume admission; fresh observations are still required revoke Pause, stop the tunnel, and rotate its local access credential + bridge enable/disable/status/pair/revoke-client/probe/jobs/get/cancel pair init / add / list / revoke: pair devices for the optional gateway mcp Stdio bridge for local MCP testing (not official Dot acceptance) probe Local API diagnostics with JSON on stdin @@ -322,6 +325,9 @@ func (unavailableDriver) Execute(context.Context, model.Action, model.View, stri func (unavailableDriver) Release(context.Context) error { return nil } func serveLegacy(ctx context.Context, dir string, c config.Config, v vault.Vault, args []string) error { + if c.DotBridgeEnabled { + return errors.New("task_bridge_gateway_unsupported") + } f := flags("serve") gatewayMode := f.Bool("gateway", false, "Serve the explicitly paired device directory") withoutTunnel := f.Bool("no-tunnel", false, "Local development/diagnostics only") diff --git a/cmd/dotcontrol/runtime.go b/cmd/dotcontrol/runtime.go index 12fe80b..25dd0e2 100644 --- a/cmd/dotcontrol/runtime.go +++ b/cmd/dotcontrol/runtime.go @@ -22,6 +22,7 @@ import ( "github.com/dotcontrol/dotcontrol/internal/config" "github.com/dotcontrol/dotcontrol/internal/connections" "github.com/dotcontrol/dotcontrol/internal/desktop" + "github.com/dotcontrol/dotcontrol/internal/dotbridge" "github.com/dotcontrol/dotcontrol/internal/driver" "github.com/dotcontrol/dotcontrol/internal/engine" "github.com/dotcontrol/dotcontrol/internal/gateway" @@ -54,6 +55,7 @@ type runtimeOwner struct { vault vault.Vault eng *engine.Engine bridge *codexbridge.Bridge + dotBridge *dotbridge.Service monitor *activity.Managed native *driver.Cua sup *tunnel.Supervisor @@ -179,6 +181,28 @@ func serve(ctx context.Context, dir string, c config.Config, v vault.Vault, args return errors.New("revocation_state_unavailable") } owner := &runtimeOwner{ctx: ctx, stop: cancel, dir: dir, cfg: c, vault: v, eng: eng, monitor: monitor, sup: &tunnel.Supervisor{}, revoked: revErr == nil, noTunnel: *noTunnel, repair: make(chan struct{}, 1)} + if c.DotBridgeEnabled { + if err := dotbridge.EnsureStateKey(v); err != nil { + return err + } + dotBridge, err := dotbridge.Open(ctx, dotbridge.Config{ + Root: filepath.Join(dir, "dot-bridge"), + Listen: c.Listen, + Vault: v, + MCPQueue: c.DeviceID, + Paused: func() bool { return eng.Device().Paused }, + Revoked: func() bool { + owner.mu.Lock() + rev := owner.revoked + owner.mu.Unlock() + return rev + }, + }) + if err != nil { + return err + } + owner.dotBridge = dotBridge + } if owner.revoked { _ = eng.Pause(true) } @@ -229,7 +253,17 @@ func serve(ctx context.Context, dir string, c config.Config, v vault.Vault, args } mux := http.NewServeMux() mux.Handle("/onboard/", ui) - mux.Handle("/mcp", guard(mcpserver.HTTP(observedBackend{owner, upgrade}))) + mcpHandler := mcpserver.HTTP(observedBackend{owner, upgrade}) + if owner.dotBridge != nil { + mcpHandler, err = mcpserver.HTTPWithBridge(observedBackend{owner, upgrade}, owner.dotBridge) + if err != nil { + return err + } + } + mux.Handle("/mcp", guard(mcpHandler)) + if owner.dotBridge != nil { + mux.Handle("/dot-bridge/", owner.dotBridge.Handler()) + } // The existing authenticated RPC remains separate from browser sessions. mux.Handle("/rpc", guard(gateway.Handler(upgrade))) mux.Handle("/admin/", localapi.Guard(adminToken, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -269,6 +303,68 @@ func serve(ctx context.Context, dir string, c config.Config, v vault.Vault, args out, e = owner.Action(r.Context(), "repair") case "/admin/pause", "/admin/resume", "/admin/revoke": out, e = owner.Action(r.Context(), strings.TrimPrefix(r.URL.Path, "/admin/")) + case "/admin/bridge/enable": + if owner.dotBridge == nil { + e = errors.New("bridge_unavailable") + } else { + e = owner.dotBridge.Enable() + if e == nil { + out = map[string]bool{"enabled": true} + } + } + case "/admin/bridge/disable": + if owner.dotBridge == nil { + e = errors.New("bridge_unavailable") + } else { + e = owner.dotBridge.Disable() + out = map[string]bool{"enabled": false} + } + case "/admin/bridge/status": + if owner.dotBridge == nil { + e = errors.New("bridge_unavailable") + } else { + out, e = owner.dotBridge.AdminStatus() + } + case "/admin/bridge/pair": + if owner.dotBridge == nil { + e = errors.New("bridge_unavailable") + } else { + file, path, err := owner.dotBridge.Pair("") + if err != nil { + e = err + } else { + out = map[string]any{"pairing_file": path, "client_id": file.ClientID, "queue_id": file.QueueID} + } + } + case "/admin/bridge/revoke-client": + var input struct { + ClientID string `json:"client_id"` + } + r.Body = http.MaxBytesReader(w, r.Body, 1<<20) + d := json.NewDecoder(r.Body) + d.DisallowUnknownFields() + if owner.dotBridge == nil { + e = errors.New("bridge_unavailable") + } else if d.Decode(&input) != nil || input.ClientID == "" { + e = errors.New("invalid_request") + } else { + e = owner.dotBridge.RevokeClient(input.ClientID) + out = map[string]bool{"revoked": e == nil} + } + case "/admin/bridge/probe": + var input struct { + ClientID string `json:"client_id"` + } + r.Body = http.MaxBytesReader(w, r.Body, 1<<20) + d := json.NewDecoder(r.Body) + d.DisallowUnknownFields() + if owner.dotBridge == nil { + e = errors.New("bridge_unavailable") + } else if d.Decode(&input) != nil || input.ClientID == "" { + e = errors.New("invalid_request") + } else { + out, e = owner.dotBridge.StartProbe(input.ClientID) + } default: http.NotFound(w, r) return @@ -295,6 +391,9 @@ func serve(ctx context.Context, dir string, c config.Config, v vault.Vault, args // Official turns must finish their bounded interruption/reconciliation // before the shared engine is closed. Unconfirmed stops remain journaled. bridge.Close() + if owner.dotBridge != nil { + _ = owner.dotBridge.Close() + } _ = eng.Close() <-workerDone owner.mu.Lock() diff --git a/docs/DOT_BRIDGE.md b/docs/DOT_BRIDGE.md new file mode 100644 index 0000000..85fcdee --- /dev/null +++ b/docs/DOT_BRIDGE.md @@ -0,0 +1,19 @@ +# Dot bridge + +The Dot bridge is opt-in via `dot_bridge_enabled` in local config. It does not change the default 13-tool MCP catalog unless the runtime is started with bridge support enabled. + +## Operator flow + +1. Enable bridge support in config and start `dotcontrol serve`. +2. Run `dotcontrol bridge enable`. +3. Run `dotcontrol bridge pair` and import the secure pairing file into Magpie. +4. Connect the private Dot plugin, refresh MCP metadata, and subscribe to `dot.task.available`. +5. Run `dotcontrol bridge probe ` and complete the probe job through MCP claim/complete. +6. Poll `GET /dot-bridge/v1/status` with the paired Bearer token. + +## Security notes + +- Loopback client API uses Bearer tokens only; queue scope is resolved from the paired token. +- MCP task tools bind to the configured bridge queue, not arbitrary client input. +- Readiness is derived from a real probe claim/completion lifecycle; there is no manual readiness setter. +- The bridge cannot cryptographically prove that Dot itself completed the probe versus another authorized MCP caller using the same tunnel credential. diff --git a/internal/config/config.go b/internal/config/config.go index 390cc63..6dd333c 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -30,6 +30,7 @@ type Config struct { TerminalProfileID string `json:"terminal_profile_id,omitempty"` TerminalServerID string `json:"terminal_server_id,omitempty"` TerminalConfigHash string `json:"terminal_config_hash,omitempty"` + DotBridgeEnabled bool `json:"dot_bridge_enabled,omitempty"` } // Network affects only the official tunnel control plane. ProxyURL is never a diff --git a/internal/dotbridge/adapters.go b/internal/dotbridge/adapters.go new file mode 100644 index 0000000..62d4e2e --- /dev/null +++ b/internal/dotbridge/adapters.go @@ -0,0 +1,79 @@ +package dotbridge + +import ( + "context" + + "github.com/dotcontrol/dotcontrol/internal/dotevents" + "github.com/dotcontrol/dotcontrol/internal/dotqueue" +) + +// QueueAPI isolates Task3 integration from dotqueue store details. +type QueueAPI interface { + Initialize(enabled bool) error + SetEnabled(enabled bool) error + ConfigStatus() (dotqueue.ConfigStatus, error) + Submit(clientID string, sub dotqueue.Submission) (dotqueue.Job, error) + Get(clientID, jobID string) (dotqueue.Job, error) + ByRequest(clientID, requestID string) (dotqueue.Job, error) + Claim(queueID, jobID, workerID string) (dotqueue.Claim, error) + Progress(jobID, token, state, summary string) (dotqueue.Job, error) + Complete(jobID, token string, result dotqueue.Result) (dotqueue.Job, error) + Cancel(clientID, jobID string) (dotqueue.Job, error) + Recover() error + Pending(queueID string) ([]dotqueue.Notice, error) +} + +// EventsAPI isolates Task3 integration from dotevents manager details. +type EventsAPI interface { + SetEnabled(enabled bool) error + Subscribe(ctx context.Context, principal string, params dotevents.Params) (dotevents.Subscription, error) + Unsubscribe(ctx context.Context, principal string, params dotevents.Params) error + List(principal string) []dotevents.Definition + Deliver(ctx context.Context, principal, queueID string, notice dotevents.Notice) error + Status(principal, queueID string) (dotevents.Status, error) +} + +type queueAdapter struct{ *dotqueue.Store } + +type eventsAdapter struct{ *dotevents.Manager } + +func wrapQueue(s *dotqueue.Store) QueueAPI { return queueAdapter{s} } +func wrapEvents(m *dotevents.Manager) EventsAPI { return eventsAdapter{m} } + +func (q queueAdapter) Initialize(enabled bool) error { return q.Store.Initialize(enabled) } +func (q queueAdapter) SetEnabled(enabled bool) error { return q.Store.SetEnabled(enabled) } +func (q queueAdapter) ConfigStatus() (dotqueue.ConfigStatus, error) { return q.Store.ConfigStatus() } +func (q queueAdapter) Submit(clientID string, sub dotqueue.Submission) (dotqueue.Job, error) { + return q.Store.Submit(clientID, sub) +} +func (q queueAdapter) Get(clientID, jobID string) (dotqueue.Job, error) { return q.Store.Get(clientID, jobID) } +func (q queueAdapter) ByRequest(clientID, requestID string) (dotqueue.Job, error) { + return q.Store.ByRequest(clientID, requestID) +} +func (q queueAdapter) Claim(queueID, jobID, workerID string) (dotqueue.Claim, error) { + return q.Store.Claim(queueID, jobID, workerID) +} +func (q queueAdapter) Progress(jobID, token, state, summary string) (dotqueue.Job, error) { + return q.Store.Progress(jobID, token, state, summary) +} +func (q queueAdapter) Complete(jobID, token string, result dotqueue.Result) (dotqueue.Job, error) { + return q.Store.Complete(jobID, token, result) +} +func (q queueAdapter) Cancel(clientID, jobID string) (dotqueue.Job, error) { return q.Store.Cancel(clientID, jobID) } +func (q queueAdapter) Recover() error { return q.Store.Recover() } +func (q queueAdapter) Pending(queueID string) ([]dotqueue.Notice, error) { return q.Store.Pending(queueID) } + +func (e eventsAdapter) SetEnabled(enabled bool) error { return e.Manager.SetEnabled(enabled) } +func (e eventsAdapter) Subscribe(ctx context.Context, principal string, params dotevents.Params) (dotevents.Subscription, error) { + return e.Manager.Subscribe(ctx, principal, params) +} +func (e eventsAdapter) Unsubscribe(ctx context.Context, principal string, params dotevents.Params) error { + return e.Manager.Unsubscribe(ctx, principal, params) +} +func (e eventsAdapter) List(principal string) []dotevents.Definition { return e.Manager.List(principal) } +func (e eventsAdapter) Deliver(ctx context.Context, principal, queueID string, notice dotevents.Notice) error { + return e.Manager.Deliver(ctx, principal, queueID, notice) +} +func (e eventsAdapter) Status(principal, queueID string) (dotevents.Status, error) { + return e.Manager.Status(principal, queueID) +} diff --git a/internal/dotbridge/api.go b/internal/dotbridge/api.go new file mode 100644 index 0000000..2450ac7 --- /dev/null +++ b/internal/dotbridge/api.go @@ -0,0 +1,182 @@ +package dotbridge + +import ( + "encoding/json" + "errors" + "io" + "net" + "net/http" + "net/netip" + "strings" +) + +const maxBody = 128 << 10 + +func (s *Service) Handler() http.Handler { + mux := http.NewServeMux() + mux.HandleFunc("/dot-bridge/v1/status", s.handleStatus) + mux.HandleFunc("/dot-bridge/v1/jobs", s.handleJobs) + mux.HandleFunc("/dot-bridge/v1/jobs/", s.handleJobByID) + mux.HandleFunc("/dot-bridge/v1/requests/", s.handleRequestByID) + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + host, _, err := net.SplitHostPort(r.Host) + if err != nil { + host = r.Host + } + ip, err := netip.ParseAddr(host) + if err != nil || !ip.IsLoopback() || r.Header.Get("Origin") != "" { + writeError(w, http.StatusForbidden, "invalid_origin", "") + return + } + if r.Method != http.MethodGet && r.Method != http.MethodPost { + writeError(w, http.StatusMethodNotAllowed, "method_rejected", "") + return + } + client, err := s.authenticate(r) + if err != nil { + writeError(w, http.StatusUnauthorized, "unauthorized", "") + return + } + r = r.WithContext(withClient(r.Context(), client)) + mux.ServeHTTP(w, r) + }) +} + +func (s *Service) authenticate(r *http.Request) (ClientPolicy, error) { + auth := r.Header.Get("Authorization") + if !strings.HasPrefix(auth, "Bearer ") { + return ClientPolicy{}, errors.New("unauthorized") + } + return s.clientFromToken(strings.TrimSpace(strings.TrimPrefix(auth, "Bearer "))) +} + +func (s *Service) handleStatus(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + writeError(w, http.StatusMethodNotAllowed, "method_rejected", "") + return + } + client := clientFrom(r.Context()) + status, err := s.StatusForClient(client.ClientID) + if err != nil { + writeError(w, http.StatusBadRequest, "bridge_unavailable", "") + return + } + writeJSON(w, http.StatusOK, status) +} + +func (s *Service) handleJobs(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + writeError(w, http.StatusMethodNotAllowed, "method_rejected", "") + return + } + client := clientFrom(r.Context()) + req, err := decodeSubmit(r.Body) + if err != nil { + writeError(w, http.StatusBadRequest, "invalid_request", "") + return + } + job, code, err := s.submit(client, req) + if err != nil { + writeError(w, http.StatusBadRequest, mapError(err), job.JobID) + return + } + writeJSON(w, code, job) +} + +func (s *Service) handleJobByID(w http.ResponseWriter, r *http.Request) { + client := clientFrom(r.Context()) + jobID := strings.TrimPrefix(r.URL.Path, "/dot-bridge/v1/jobs/") + if jobID == "" || strings.Contains(jobID, "/") { + writeError(w, http.StatusNotFound, "notfound", "") + return + } + if strings.HasSuffix(r.URL.Path, "/cancel") { + if r.Method != http.MethodPost { + writeError(w, http.StatusMethodNotAllowed, "method_rejected", "") + return + } + jobID = strings.TrimSuffix(jobID, "/cancel") + job, err := s.cancelJob(client, jobID) + if err != nil { + writeError(w, http.StatusBadRequest, mapError(err), jobID) + return + } + writeJSON(w, http.StatusOK, job) + return + } + if r.Method != http.MethodGet { + writeError(w, http.StatusMethodNotAllowed, "method_rejected", "") + return + } + job, err := s.getJob(client, jobID) + if err != nil { + writeError(w, http.StatusNotFound, mapError(err), jobID) + return + } + writeJSON(w, http.StatusOK, job) +} + +func (s *Service) handleRequestByID(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + writeError(w, http.StatusMethodNotAllowed, "method_rejected", "") + return + } + client := clientFrom(r.Context()) + requestID := strings.TrimPrefix(r.URL.Path, "/dot-bridge/v1/requests/") + if requestID == "" || strings.Contains(requestID, "/") { + writeError(w, http.StatusNotFound, "notfound", "") + return + } + job, err := s.byRequest(client, requestID) + if err != nil { + writeError(w, http.StatusNotFound, mapError(err), "") + return + } + writeJSON(w, http.StatusOK, job) +} + +func decodeSubmit(body io.Reader) (SubmitRequest, error) { + dec := json.NewDecoder(io.LimitReader(body, maxBody)) + dec.DisallowUnknownFields() + var req SubmitRequest + if dec.Decode(&req) != nil { + return SubmitRequest{}, errors.New("invalid_request") + } + if dec.Decode(&struct{}{}) != nil { + return SubmitRequest{}, errors.New("invalid_request") + } + return req, nil +} + +func writeJSON(w http.ResponseWriter, code int, v any) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(code) + _ = json.NewEncoder(w).Encode(v) +} + +func writeError(w http.ResponseWriter, code int, errCode, jobID string) { + writeJSON(w, code, ErrorDTO{Error: ErrorBody{Code: errCode, JobID: jobID}}) +} + +func mapError(err error) string { + if err == nil { + return "unknown" + } + msg := err.Error() + switch { + case strings.Contains(msg, "probe_required"): + return "probe_required" + case strings.Contains(msg, "bridge_disabled"): + return "bridge_disabled" + case strings.Contains(msg, "bridge_paused"): + return "bridge_paused" + case strings.Contains(msg, "access_revoked"): + return "access_revoked" + case strings.Contains(msg, "notfound"): + return "notfound" + case strings.Contains(msg, "conflict"): + return "conflict" + default: + return "unknown" + } +} diff --git a/internal/dotbridge/bridge_test.go b/internal/dotbridge/bridge_test.go new file mode 100644 index 0000000..9c290df --- /dev/null +++ b/internal/dotbridge/bridge_test.go @@ -0,0 +1,122 @@ +package dotbridge + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "path/filepath" + "testing" + "time" + + "github.com/dotcontrol/dotcontrol/internal/dotqueue" + "github.com/dotcontrol/dotcontrol/internal/mcpserver" + "github.com/dotcontrol/dotcontrol/internal/service" +) + +type noopBackend struct{} + +func (noopBackend) Call(context.Context, string, service.Request) (any, error) { + return map[string]any{"ok": true}, nil +} + +func openTestBridge(t *testing.T) (*Service, string, string) { + t.Helper() + root := t.TempDir() + vault := NewFakeVault() + if err := EnsureStateKey(vault); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + svc, err := Open(ctx, Config{ + Root: filepath.Join(root, "dot-bridge"), + Listen: "127.0.0.1:0", + Vault: vault, + MCPQueue: "queue.a", + Paused: func() bool { return false }, + Revoked: func() bool { return false }, + }) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = svc.Close() }) + if err := svc.Enable(); err != nil { + t.Fatal(err) + } + _, path, err := svc.Pair("client.a") + if err != nil { + t.Fatal(err) + } + var file PairingFile + if json.Unmarshal(mustRead(path), &file) != nil { + t.Fatal("pairing file invalid") + } + return svc, file.Token, file.ClientID +} + +func mustRead(path string) []byte { + data, err := readFile(path) + if err != nil { + panic(err) + } + return data +} + +func readFile(path string) ([]byte, error) { + return readFileOS(path) +} + +func TestBridgeAPIProbeClaimComplete(t *testing.T) { + svc, token, clientID := openTestBridge(t) + probe, err := svc.StartProbe(clientID) + if err != nil { + t.Fatal(err) + } + claim, err := svc.ClaimTask(probe.QueueID, probe.JobID, "worker.1") + if err != nil { + t.Fatal(err) + } + job, err := svc.CompleteTask(probe.JobID, claim.Token, dotqueue.Result{Text: mustProbeAnswer(svc, clientID)}) + if err != nil || job.State != dotqueue.StateCompleted { + t.Fatalf("complete: %v %#v", err, job) + } + status, err := svc.StatusForClient(clientID) + if err != nil || !status.Ready { + t.Fatalf("ready status %#v err=%v", status, err) + } + handler := svc.Handler() + req := httptest.NewRequest(http.MethodGet, "/dot-bridge/v1/status", nil) + req.Header.Set("Authorization", "Bearer "+token) + req.Host = "127.0.0.1:9317" + res := httptest.NewRecorder() + handler.ServeHTTP(res, req) + if res.Code != http.StatusOK { + t.Fatalf("status code %d", res.Code) + } +} + +func TestMCPAddsBridgeTools(t *testing.T) { + svc, _, _ := openTestBridge(t) + s, err := mcpserver.NewWithBridge(noopBackend{}, svc) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + client := mcpTestClient(t, s, ctx) + list, err := client.ListTools(ctx, nil) + if err != nil { + t.Fatal(err) + } + if len(list.Tools) != 17 { + t.Fatalf("tool count=%d", len(list.Tools)) + } +} + +func mustProbeAnswer(svc *Service, clientID string) string { + svc.mu.Lock() + answer := svc.state.Clients[clientID].ProbeAnswer + svc.mu.Unlock() + return answer +} diff --git a/internal/dotbridge/context.go b/internal/dotbridge/context.go new file mode 100644 index 0000000..ba16401 --- /dev/null +++ b/internal/dotbridge/context.go @@ -0,0 +1,16 @@ +package dotbridge + +import "context" + +type clientKey struct{} + +func withClient(ctx context.Context, client ClientPolicy) context.Context { + return context.WithValue(ctx, clientKey{}, client) +} + +func clientFrom(ctx context.Context) ClientPolicy { + if v, ok := ctx.Value(clientKey{}).(ClientPolicy); ok { + return v + } + return ClientPolicy{} +} diff --git a/internal/dotbridge/fakevault.go b/internal/dotbridge/fakevault.go new file mode 100644 index 0000000..c8db140 --- /dev/null +++ b/internal/dotbridge/fakevault.go @@ -0,0 +1,34 @@ +package dotbridge + +import "sync" + +type FakeVault struct { + mu sync.Mutex + data map[string]string +} + +func NewFakeVault() *FakeVault { + return &FakeVault{data: map[string]string{}} +} + +func (f *FakeVault) Get(name string) (string, error) { + f.mu.Lock() + defer f.mu.Unlock() + if v, ok := f.data[name]; ok { + return v, nil + } + return "", errMissing +} + +func (f *FakeVault) Set(name, value string) error { + f.mu.Lock() + defer f.mu.Unlock() + f.data[name] = value + return nil +} + +var errMissing = &vaultMissing{} + +type vaultMissing struct{} + +func (v *vaultMissing) Error() string { return "not_found" } diff --git a/internal/dotbridge/mcp_test.go b/internal/dotbridge/mcp_test.go new file mode 100644 index 0000000..2340257 --- /dev/null +++ b/internal/dotbridge/mcp_test.go @@ -0,0 +1,26 @@ +package dotbridge + +import ( + "context" + "testing" + "time" + + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +func mcpTestClient(t *testing.T, s *mcp.Server, ctx context.Context) *mcp.ClientSession { + t.Helper() + client := mcp.NewClient(&mcp.Implementation{Name: "bridge-test", Version: "1"}, nil) + st, mem := mcp.NewInMemoryTransports() + sess, err := s.Connect(ctx, st, nil) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { sess.Close() }) + cs, err := client.Connect(ctx, mem, nil) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { cs.Close() }) + return cs +} diff --git a/internal/dotbridge/pairing.go b/internal/dotbridge/pairing.go new file mode 100644 index 0000000..a92358a --- /dev/null +++ b/internal/dotbridge/pairing.go @@ -0,0 +1,37 @@ +package dotbridge + +import ( + "encoding/json" + "os" + "path/filepath" + + "github.com/dotcontrol/dotcontrol/internal/vault" +) + +func writePairingFile(root string, file PairingFile) (string, error) { + if err := os.MkdirAll(root, 0700); err != nil { + return "", err + } + name := "pairing-" + file.ClientID + ".json" + path := filepath.Join(root, name) + data, err := json.MarshalIndent(file, "", " ") + if err != nil { + return "", err + } + if err := os.WriteFile(path, data, 0600); err != nil { + return "", err + } + return path, nil +} + +func EnsureStateKey(v vault.Store) error { + _, err := v.Get(StateKeyName) + if err == nil { + return nil + } + encoded, err := randomStateKey() + if err != nil { + return err + } + return v.Set(StateKeyName, encoded) +} diff --git a/internal/dotbridge/probe.go b/internal/dotbridge/probe.go new file mode 100644 index 0000000..4880769 --- /dev/null +++ b/internal/dotbridge/probe.go @@ -0,0 +1,68 @@ +package dotbridge + +import ( + "crypto/rand" + "encoding/hex" + "errors" + "fmt" + "time" + + "github.com/dotcontrol/dotcontrol/internal/dotqueue" +) + +type ProbeResult struct { + JobID string `json:"job_id"` + RequestID string `json:"request_id"` + QueueID string `json:"queue_id"` + Steps []string `json:"steps"` +} + +func (s *Service) StartProbe(clientID string) (ProbeResult, error) { + s.mu.Lock() + policy, ok := s.state.Clients[clientID] + s.mu.Unlock() + if !ok || policy.Revoked { + return ProbeResult{}, errors.New("client_not_found") + } + if err := s.gate(); err != nil { + return ProbeResult{}, err + } + var nonce [16]byte + if _, err := rand.Read(nonce[:]); err != nil { + return ProbeResult{}, err + } + answer := hex.EncodeToString(nonce[:]) + requestID := "probe." + time.Now().UTC().Format("20060102150405") + sub := dotqueue.Submission{ + RequestID: requestID, + Mode: dotqueue.ModeAnswer, + SessionID: "probe", + Messages: []dotqueue.Message{{ + Role: "user", + Content: fmt.Sprintf("Reply with exactly this nonce: %s", answer), + }}, + } + job, err := s.queue.Submit(clientID, sub) + if err != nil { + return ProbeResult{}, err + } + s.mu.Lock() + policy.ProbeJobID = job.ID + policy.ProbeNonce = answer + policy.ProbeAnswer = answer + policy.ProbeVerified = time.Time{} + policy.ProbeExpires = time.Time{} + s.state.Clients[clientID] = policy + _ = s.saveState() + s.mu.Unlock() + return ProbeResult{ + JobID: job.ID, + RequestID: requestID, + QueueID: policy.QueueID, + Steps: []string{ + "Connect the private Dot plugin to this DotControl runtime.", + "Refresh MCP metadata and subscribe to dot.task.available for this queue.", + "Claim the probe job with dot_task_claim, then complete it with the exact nonce using dot_task_complete.", + }, + }, nil +} diff --git a/internal/dotbridge/secrets.go b/internal/dotbridge/secrets.go new file mode 100644 index 0000000..820ad20 --- /dev/null +++ b/internal/dotbridge/secrets.go @@ -0,0 +1,46 @@ +package dotbridge + +import ( + "crypto/rand" + "crypto/sha256" + "crypto/subtle" + "encoding/base64" + "encoding/hex" + "errors" +) + +func randomToken() (string, error) { + var b [32]byte + if _, err := rand.Read(b[:]); err != nil { + return "", err + } + return hex.EncodeToString(b[:]), nil +} + +func randomStateKey() (string, error) { + var b [32]byte + if _, err := rand.Read(b[:]); err != nil { + return "", err + } + return base64.StdEncoding.EncodeToString(b[:]), nil +} + +func decodeStateKey(encoded string) ([]byte, error) { + key, err := base64.StdEncoding.DecodeString(encoded) + if err != nil || len(key) != 32 { + return nil, errors.New("bridge_state_key_invalid") + } + return key, nil +} + +func hashToken(token string) string { + sum := sha256.Sum256([]byte(token)) + return hex.EncodeToString(sum[:]) +} + +func tokenMatches(hash, token string) bool { + if hash == "" || token == "" { + return false + } + return subtle.ConstantTimeCompare([]byte(hash), []byte(hashToken(token))) == 1 +} diff --git a/internal/dotbridge/service.go b/internal/dotbridge/service.go new file mode 100644 index 0000000..05613a1 --- /dev/null +++ b/internal/dotbridge/service.go @@ -0,0 +1,427 @@ +package dotbridge + +import ( + "context" + "encoding/json" + "errors" + "os" + "path/filepath" + "sync" + "time" + + "github.com/dotcontrol/dotcontrol/internal/dotevents" + "github.com/dotcontrol/dotcontrol/internal/dotqueue" + "github.com/dotcontrol/dotcontrol/internal/vault" +) + +// Service owns bridge lifecycle, local API auth, and delivery worker wiring. +type Service struct { + mu sync.Mutex + ctx context.Context + cancel context.CancelFunc + root string + listen string + vault vault.Store + queue QueueAPI + events EventsAPI + mcpQueue string + mcpEpoch uint64 + state PersistedState + recovered bool + worker *worker + paused func() bool + revoked func() bool +} + +type Config struct { + Root string + Listen string + Vault vault.Store + MCPQueue string + MCPEpoch uint64 + Paused func() bool + Revoked func() bool +} + +func Open(ctx context.Context, cfg Config) (*Service, error) { + if cfg.Vault == nil || cfg.Root == "" || cfg.Listen == "" { + return nil, errors.New("bridge_config_invalid") + } + encoded, err := cfg.Vault.Get(StateKeyName) + if err != nil { + return nil, errors.New("bridge_state_key_missing") + } + key, err := decodeStateKey(encoded) + if err != nil { + return nil, err + } + queueDir := filepath.Join(cfg.Root, "queue") + eventsDir := filepath.Join(cfg.Root, "events") + queueStore, err := dotqueue.Open(queueDir, key) + if err != nil { + return nil, err + } + eventsMgr, err := dotevents.Open(eventsDir, key, &dotevents.Delivery{}) + if err != nil { + return nil, err + } + ctx, cancel := context.WithCancel(ctx) + s := &Service{ + ctx: ctx, + cancel: cancel, + root: cfg.Root, + listen: cfg.Listen, + vault: cfg.Vault, + queue: wrapQueue(queueStore), + events: wrapEvents(eventsMgr), + mcpQueue: cfg.MCPQueue, + mcpEpoch: cfg.MCPEpoch, + paused: cfg.Paused, + revoked: cfg.Revoked, + state: PersistedState{Version: APIVersion, Clients: map[string]ClientPolicy{}}, + } + if err := s.loadState(); err != nil { + cancel() + return nil, err + } + if err := s.recoverOnce(); err != nil { + cancel() + return nil, err + } + s.worker = newWorker(s) + s.worker.Start() + return s, nil +} + +func (s *Service) Close() error { + if s.cancel != nil { + s.cancel() + } + if s.worker != nil { + s.worker.Wait() + } + return nil +} + +func (s *Service) recoverOnce() error { + s.mu.Lock() + defer s.mu.Unlock() + if s.recovered { + return nil + } + if err := s.queue.Recover(); err != nil { + return err + } + s.recovered = true + return nil +} + +func (s *Service) gate() error { + if s.revoked != nil && s.revoked() { + return errors.New("access_revoked") + } + if s.paused != nil && s.paused() { + return errors.New("bridge_paused") + } + if !s.state.Enabled { + return errors.New("bridge_disabled") + } + return nil +} + +func (s *Service) Enable() error { + s.mu.Lock() + defer s.mu.Unlock() + if err := s.queue.Initialize(false); err != nil { + return err + } + if err := s.queue.SetEnabled(true); err != nil { + return err + } + if err := s.events.SetEnabled(true); err != nil { + return err + } + s.state.Enabled = true + return s.saveState() +} + +func (s *Service) Disable() error { + s.mu.Lock() + defer s.mu.Unlock() + if err := s.queue.SetEnabled(false); err != nil { + return err + } + if err := s.events.SetEnabled(false); err != nil { + return err + } + s.state.Enabled = false + return s.saveState() +} + +func (s *Service) AdminStatus() (map[string]any, error) { + s.mu.Lock() + defer s.mu.Unlock() + paired := 0 + for _, policy := range s.state.Clients { + if !policy.Revoked { + paired++ + } + } + return map[string]any{ + "version": APIVersion, + "enabled": s.state.Enabled, + "paired": paired > 0, + "clients": paired, + }, nil +} + +func (s *Service) StatusForClient(clientID string) (StatusDTO, error) { + s.mu.Lock() + defer s.mu.Unlock() + policy, ok := s.state.Clients[clientID] + if !ok || policy.Revoked { + return StatusDTO{Version: APIVersion, Enabled: s.state.Enabled, Paired: false, Ready: false, Code: "not_paired", Capabilities: CapPolicy{}}, nil + } + return s.statusDTO(policy), nil +} + +func (s *Service) statusDTO(policy ClientPolicy) StatusDTO { + now := time.Now().UTC() + eventsStatus, _ := s.events.Status(policy.QueueID, policy.QueueID) + sub := SubscriptionDTO{State: "inactive"} + if eventsStatus.Subscribed && eventsStatus.CallbackVerified { + sub.State = "active" + } + ready := false + code := "probe_required" + var verifiedAt, expiresAt *time.Time + if s.state.Enabled && policy.ProbeVerified.After(time.Time{}) && policy.ProbeExpires.After(now) { + ready = eventsStatus.Subscribed && eventsStatus.CallbackVerified + if ready { + code = "ready" + verifiedAt = &policy.ProbeVerified + expiresAt = &policy.ProbeExpires + if sub.State == "active" { + sub.ExpiresAt = expiresAt + } + } + } + return StatusDTO{ + Version: APIVersion, + Enabled: s.state.Enabled, + Paired: true, + Ready: ready, + Code: code, + VerifiedAt: verifiedAt, + ExpiresAt: expiresAt, + Subscription: sub, + Capabilities: policy.Capabilities, + } +} + +func (s *Service) Pair(clientID string) (PairingFile, string, error) { + if clientID == "" { + clientID = "client." + time.Now().UTC().Format("20060102150405") + } + token, err := randomToken() + if err != nil { + return PairingFile{}, "", err + } + s.mu.Lock() + defer s.mu.Unlock() + policy := ClientPolicy{ + ClientID: clientID, + QueueID: clientID, + TokenHash: hashToken(token), + CreatedAt: time.Now().UTC(), + Capabilities: CapPolicy{Text: false, Tasks: false}, + } + s.state.Clients[clientID] = policy + if err := s.saveState(); err != nil { + return PairingFile{}, "", err + } + file := PairingFile{Version: APIVersion, ClientID: clientID, QueueID: clientID, Token: token, Listen: s.listen, CreatedAt: policy.CreatedAt} + path, err := writePairingFile(s.root, file) + return file, path, err +} + +func (s *Service) RevokeClient(clientID string) error { + s.mu.Lock() + defer s.mu.Unlock() + policy, ok := s.state.Clients[clientID] + if !ok { + return errors.New("client_not_found") + } + policy.Revoked = true + s.state.Clients[clientID] = policy + return s.saveState() +} + +func (s *Service) clientFromToken(token string) (ClientPolicy, error) { + s.mu.Lock() + defer s.mu.Unlock() + for _, policy := range s.state.Clients { + if policy.Revoked { + continue + } + if tokenMatches(policy.TokenHash, token) { + return policy, nil + } + } + return ClientPolicy{}, errors.New("unauthorized") +} + +func (s *Service) MCPPrincipal() (string, string, error) { + if s.mcpQueue == "" { + return "", "", errors.New("mcp_queue_unconfigured") + } + return s.mcpQueue, s.mcpQueue, nil +} + +func (s *Service) EventsList(principal string) []dotevents.Definition { + return s.events.List(principal) +} + +func (s *Service) EventsSubscribe(ctx context.Context, principal string, params dotevents.Params) (dotevents.Subscription, error) { + if err := s.gate(); err != nil { + return dotevents.Subscription{}, err + } + return s.events.Subscribe(ctx, principal, params) +} + +func (s *Service) EventsUnsubscribe(ctx context.Context, principal string, params dotevents.Params) error { + if err := s.gate(); err != nil { + return err + } + return s.events.Unsubscribe(ctx, principal, params) +} + +func (s *Service) ClaimTask(queueID, jobID, workerID string) (dotqueue.Claim, error) { + return s.claimTask(queueID, jobID, workerID) +} + +func (s *Service) ProgressTask(jobID, token, state, summary string) (dotqueue.Job, error) { + return s.progressTask(jobID, token, state, summary) +} + +func (s *Service) CompleteTask(jobID, token string, result dotqueue.Result) (dotqueue.Job, error) { + return s.completeTask(jobID, token, result) +} + +func (s *Service) TaskStatus(principal, jobID, token string) (dotqueue.Job, error) { + return s.taskStatus(principal, jobID, token) +} + +func (s *Service) loadState() error { + path := filepath.Join(s.root, "bridge.state.json") + data, err := os.ReadFile(path) + if os.IsNotExist(err) { + return nil + } + if err != nil { + return err + } + var st PersistedState + if json.Unmarshal(data, &st) != nil { + return errors.New("bridge_state_corrupt") + } + if st.Clients == nil { + st.Clients = map[string]ClientPolicy{} + } + s.state = st + return nil +} + +func (s *Service) saveState() error { + if err := os.MkdirAll(s.root, 0700); err != nil { + return err + } + data, err := json.MarshalIndent(s.state, "", " ") + if err != nil { + return err + } + path := filepath.Join(s.root, "bridge.state.json") + return os.WriteFile(path, data, 0600) +} + +func (s *Service) submit(client ClientPolicy, req SubmitRequest) (JobDTO, int, error) { + if err := s.gate(); err != nil { + return JobDTO{}, 0, err + } + status := s.statusDTO(policy) + if !status.Ready && req.Mode != dotqueue.ModeAnswer { + return JobDTO{}, 0, errors.New("probe_required") + } + if !status.Ready { + return JobDTO{}, 0, errors.New("probe_required") + } + sub := dotqueue.Submission{ + RequestID: req.RequestID, + Mode: req.Mode, + SessionID: req.SessionID, + Task: req.Task, + } + for _, m := range req.Messages { + sub.Messages = append(sub.Messages, dotqueue.Message{Role: m.Role, Content: m.Content}) + } + job, err := s.queue.Submit(client.ClientID, sub) + if err != nil { + return JobDTO{}, 0, err + } + dto := toJobDTO(job) + dto.Accepted = true + return dto, 202, nil +} + +func (s *Service) getJob(client ClientPolicy, jobID string) (JobDTO, error) { + if err := s.gate(); err != nil { + return JobDTO{}, err + } + job, err := s.queue.Get(client.ClientID, jobID) + if err != nil { + return JobDTO{}, err + } + return toJobDTO(job), nil +} + +func (s *Service) byRequest(client ClientPolicy, requestID string) (JobDTO, error) { + if err := s.gate(); err != nil { + return JobDTO{}, err + } + job, err := s.queue.ByRequest(client.ClientID, requestID) + if err != nil { + return JobDTO{}, err + } + return toJobDTO(job), nil +} + +func (s *Service) cancelJob(client ClientPolicy, jobID string) (JobDTO, error) { + if err := s.gate(); err != nil { + return JobDTO{}, err + } + job, err := s.queue.Cancel(client.ClientID, jobID) + if err != nil { + return JobDTO{}, err + } + return toJobDTO(job), nil +} + +func toJobDTO(job dotqueue.Job) JobDTO { + dto := JobDTO{ + Version: APIVersion, + JobID: job.ID, + RequestID: job.RequestID, + RequestDigest: job.RequestDigest, + Mode: job.Mode, + SessionID: job.SessionID, + State: job.State, + CreatedAt: job.CreatedAt, + UpdatedAt: job.UpdatedAt, + CancelRequested: job.CancelRequested, + ErrorCode: job.ErrorCode, + } + if job.Result != nil { + dto.Result = &ResultDTO{Text: job.Result.Text, Cancelled: job.Result.Cancelled, ErrorCode: job.Result.ErrorCode} + } + return dto +} diff --git a/internal/dotbridge/testutil.go b/internal/dotbridge/testutil.go new file mode 100644 index 0000000..aaaf967 --- /dev/null +++ b/internal/dotbridge/testutil.go @@ -0,0 +1,7 @@ +package dotbridge + +import "os" + +func readFileOS(path string) ([]byte, error) { + return os.ReadFile(path) +} diff --git a/internal/dotbridge/types.go b/internal/dotbridge/types.go new file mode 100644 index 0000000..fc524d1 --- /dev/null +++ b/internal/dotbridge/types.go @@ -0,0 +1,105 @@ +package dotbridge + +import "time" + +const ( + APIVersion = 1 + StateKeyName = "dot-bridge-state-key" + probeValidity = 10 * time.Minute + statusCacheMaxAge = 10 * time.Second +) + +type ClientPolicy struct { + ClientID string `json:"client_id"` + QueueID string `json:"queue_id"` + TokenHash string `json:"token_hash"` + CreatedAt time.Time `json:"created_at"` + Revoked bool `json:"revoked"` + Capabilities CapPolicy `json:"capabilities"` + ProbeJobID string `json:"probe_job_id,omitempty"` + ProbeNonce string `json:"probe_nonce,omitempty"` + ProbeAnswer string `json:"probe_answer,omitempty"` + ProbeVerified time.Time `json:"probe_verified_at,omitempty"` + ProbeExpires time.Time `json:"probe_expires_at,omitempty"` +} + +type CapPolicy struct { + Text bool `json:"text"` + Tasks bool `json:"tasks"` +} + +type PersistedState struct { + Version int `json:"version"` + Enabled bool `json:"enabled"` + Clients map[string]ClientPolicy `json:"clients"` +} + +type StatusDTO struct { + Version int `json:"version"` + Enabled bool `json:"enabled"` + Paired bool `json:"paired"` + Ready bool `json:"ready"` + Code string `json:"code"` + VerifiedAt *time.Time `json:"verified_at,omitempty"` + ExpiresAt *time.Time `json:"expires_at,omitempty"` + Subscription SubscriptionDTO `json:"subscription"` + Capabilities CapPolicy `json:"capabilities"` +} + +type SubscriptionDTO struct { + State string `json:"state"` + ExpiresAt *time.Time `json:"expires_at,omitempty"` +} + +type JobDTO struct { + Version int `json:"version"` + JobID string `json:"job_id"` + RequestID string `json:"request_id"` + RequestDigest string `json:"request_digest"` + Mode string `json:"mode"` + SessionID string `json:"session_id"` + State string `json:"state"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + CancelRequested bool `json:"cancel_requested"` + Result *ResultDTO `json:"result,omitempty"` + ErrorCode string `json:"error_code,omitempty"` + Accepted bool `json:"accepted,omitempty"` +} + +type ResultDTO struct { + Text string `json:"text"` + Cancelled bool `json:"cancelled,omitempty"` + ErrorCode string `json:"error_code,omitempty"` +} + +type SubmitRequest struct { + RequestID string `json:"request_id"` + SessionID string `json:"session_id"` + Mode string `json:"mode"` + Messages []MessageDTO `json:"messages,omitempty"` + Task string `json:"task,omitempty"` +} + +type MessageDTO struct { + Role string `json:"role"` + Content string `json:"content"` +} + +type ErrorDTO struct { + Error ErrorBody `json:"error"` +} + +type ErrorBody struct { + Code string `json:"code"` + JobID string `json:"job_id,omitempty"` +} + +type PairingFile struct { + Version int `json:"version"` + ClientID string `json:"client_id"` + QueueID string `json:"queue_id"` + Token string `json:"token"` + Listen string `json:"listen"` + CreatedAt time.Time `json:"created_at"` +} diff --git a/internal/dotbridge/worker.go b/internal/dotbridge/worker.go new file mode 100644 index 0000000..bb47027 --- /dev/null +++ b/internal/dotbridge/worker.go @@ -0,0 +1,138 @@ +package dotbridge + +import ( + "context" + "sync" + "time" + + "github.com/dotcontrol/dotcontrol/internal/dotevents" + "github.com/dotcontrol/dotcontrol/internal/dotqueue" +) + +type worker struct { + svc *Service + done chan struct{} + wg sync.WaitGroup +} + +func newWorker(s *Service) *worker { + return &worker{svc: s, done: make(chan struct{})} +} + +func (w *worker) Start() { + w.wg.Add(1) + go func() { + defer w.wg.Done() + ticker := time.NewTicker(time.Second) + defer ticker.Stop() + for { + select { + case <-w.svc.ctx.Done(): + close(w.done) + return + case <-ticker.C: + w.tick() + } + } + }() +} + +func (w *worker) Wait() { + w.wg.Wait() +} + +func (w *worker) tick() { + w.svc.mu.Lock() + queues := make([]string, 0, len(w.svc.state.Clients)) + for _, policy := range w.svc.state.Clients { + if !policy.Revoked { + queues = append(queues, policy.QueueID) + } + } + enabled := w.svc.state.Enabled + w.svc.mu.Unlock() + if !enabled { + return + } + for _, queueID := range queues { + if err := w.svc.gate(); err != nil { + return + } + notices, err := w.svc.queue.Pending(queueID) + if err != nil { + continue + } + for _, notice := range notices { + if err := w.svc.gate(); err != nil { + return + } + ctx, cancel := context.WithTimeout(w.svc.ctx, 5*time.Second) + err := w.svc.events.Deliver(ctx, queueID, queueID, dotevents.Notice{ + EventID: notice.EventID, + QueueID: notice.QueueID, + JobID: notice.JobID, + Mode: notice.Mode, + }) + cancel() + if err != nil { + continue + } + } + } +} + +func (s *Service) claimTask(queueID, jobID, workerID string) (dotqueue.Claim, error) { + if err := s.gate(); err != nil { + return dotqueue.Claim{}, err + } + authorized, _, err := s.MCPPrincipal() + if err != nil { + return dotqueue.Claim{}, err + } + if queueID != authorized { + return dotqueue.Claim{}, dotqueue.Error{Code: "unknown"} + } + return s.queue.Claim(queueID, jobID, workerID) +} + +func (s *Service) progressTask(jobID, token, state, summary string) (dotqueue.Job, error) { + if err := s.gate(); err != nil { + return dotqueue.Job{}, err + } + return s.queue.Progress(jobID, token, state, summary) +} + +func (s *Service) completeTask(jobID, token string, result dotqueue.Result) (dotqueue.Job, error) { + if err := s.gate(); err != nil { + return dotqueue.Job{}, err + } + job, err := s.queue.Complete(jobID, token, result) + if err != nil { + return job, err + } + s.mu.Lock() + for id, policy := range s.state.Clients { + if policy.ProbeJobID == jobID && policy.ProbeAnswer != "" && result.Text == policy.ProbeAnswer { + now := time.Now().UTC() + policy.ProbeVerified = now + policy.ProbeExpires = now.Add(probeValidity) + policy.Capabilities.Text = true + s.state.Clients[id] = policy + _ = s.saveState() + break + } + } + s.mu.Unlock() + return job, nil +} + +func (s *Service) taskStatus(principal, jobID, token string) (dotqueue.Job, error) { + if err := s.gate(); err != nil { + return dotqueue.Job{}, err + } + job, err := s.queue.Get(principal, jobID) + if err != nil { + return dotqueue.Job{}, err + } + return s.queue.Progress(jobID, token, job.State, job.Summary) +} diff --git a/internal/mcpserver/bridge.go b/internal/mcpserver/bridge.go new file mode 100644 index 0000000..a61b052 --- /dev/null +++ b/internal/mcpserver/bridge.go @@ -0,0 +1,159 @@ +package mcpserver + +import ( + "context" + "fmt" + "net/http" + + "github.com/dotcontrol/dotcontrol/internal/dotbridge" + "github.com/dotcontrol/dotcontrol/internal/dotevents" + "github.com/dotcontrol/dotcontrol/internal/dotqueue" + "github.com/dotcontrol/dotcontrol/internal/service" + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +type TaskClaimArgs struct { + QueueID string `json:"queue_id"` + JobID string `json:"job_id"` + WorkerID string `json:"worker_id"` +} + +type TaskProgressArgs struct { + JobID string `json:"job_id"` + Token string `json:"token"` + State string `json:"state"` + Summary string `json:"summary"` +} + +type TaskCompleteArgs struct { + JobID string `json:"job_id"` + Token string `json:"token"` + Result dotqueue.Result `json:"result"` +} + +type TaskStatusArgs struct { + JobID string `json:"job_id"` + Token string `json:"token"` +} + +type EventsListParams struct { + mcp.ParamsBase +} + +type EventsListResult struct { + mcp.ResultBase + Events []dotevents.Definition `json:"events"` +} + +type EventsSubscribeParams struct { + mcp.ParamsBase + Params dotevents.Params `json:"params"` +} + +type EventsSubscribeResult struct { + mcp.ResultBase + Subscription dotevents.Subscription `json:"subscription"` +} + +type EventsUnsubscribeParams struct { + mcp.ParamsBase + Params dotevents.Params `json:"params"` +} + +type EventsUnsubscribeResult struct { + mcp.ResultBase +} + +// NewWithBridge keeps the default 13-tool catalog and opt-in task/event methods. +func NewWithBridge(backend service.Backend, bridge *dotbridge.Service) (*mcp.Server, error) { + s := New(backend) + write := &mcp.ToolAnnotations{ReadOnlyHint: false} + read := &mcp.ToolAnnotations{ReadOnlyHint: true, IdempotentHint: true} + mcp.AddTool(s, &mcp.Tool{Name: "dot_task_claim", Description: "Claim one queued job for the authorized Dot queue. queue_id must match the configured bridge queue.", Annotations: write}, func(ctx context.Context, _ *mcp.CallToolRequest, a TaskClaimArgs) (*mcp.CallToolResult, any, error) { + claim, err := bridge.ClaimTask(a.QueueID, a.JobID, a.WorkerID) + if err != nil { + return toolErr(err), nil, nil + } + return nil, claim, nil + }) + mcp.AddTool(s, &mcp.Tool{Name: "dot_task_progress", Description: "Report progress for a claimed job using its claim token.", Annotations: write}, func(ctx context.Context, _ *mcp.CallToolRequest, a TaskProgressArgs) (*mcp.CallToolResult, any, error) { + job, err := bridge.ProgressTask(a.JobID, a.Token, a.State, a.Summary) + if err != nil { + return toolErr(err), nil, nil + } + return nil, job, nil + }) + mcp.AddTool(s, &mcp.Tool{Name: "dot_task_complete", Description: "Complete a claimed job using its claim token.", Annotations: write}, func(ctx context.Context, _ *mcp.CallToolRequest, a TaskCompleteArgs) (*mcp.CallToolResult, any, error) { + job, err := bridge.CompleteTask(a.JobID, a.Token, a.Result) + if err != nil { + return toolErr(err), nil, nil + } + return nil, job, nil + }) + mcp.AddTool(s, &mcp.Tool{Name: "dot_task_status", Description: "Read a claimed job using its claim token.", Annotations: read}, func(ctx context.Context, _ *mcp.CallToolRequest, a TaskStatusArgs) (*mcp.CallToolResult, any, error) { + principal, _, err := bridge.MCPPrincipal() + if err != nil { + return toolErr(err), nil, nil + } + job, err := bridge.TaskStatus(principal, a.JobID, a.Token) + if err != nil { + return toolErr(err), nil, nil + } + return nil, job, nil + }) + if err := mcp.AddReceivingCustomMethod(s, "events/list", func(ctx context.Context, ss *mcp.ServerSession, params *EventsListParams) (*EventsListResult, error) { + principal, _, err := bridge.MCPPrincipal() + if err != nil { + return nil, err + } + return &EventsListResult{Events: bridge.EventsList(principal)}, nil + }); err != nil { + return nil, err + } + if err := mcp.AddReceivingCustomMethod(s, "events/subscribe", func(ctx context.Context, ss *mcp.ServerSession, params *EventsSubscribeParams) (*EventsSubscribeResult, error) { + principal, queueID, err := bridge.MCPPrincipal() + if err != nil { + return nil, err + } + if params.Params.Arguments.QueueID != queueID { + return nil, fmt.Errorf("queue_mismatch") + } + sub, err := bridge.EventsSubscribe(ctx, principal, params.Params) + if err != nil { + return nil, err + } + return &EventsSubscribeResult{Subscription: sub}, nil + }); err != nil { + return nil, err + } + if err := mcp.AddReceivingCustomMethod(s, "events/unsubscribe", func(ctx context.Context, ss *mcp.ServerSession, params *EventsUnsubscribeParams) (*EventsUnsubscribeResult, error) { + principal, queueID, err := bridge.MCPPrincipal() + if err != nil { + return nil, err + } + if params.Params.Arguments.QueueID != queueID { + return nil, fmt.Errorf("queue_mismatch") + } + if err := bridge.EventsUnsubscribe(ctx, principal, params.Params); err != nil { + return nil, err + } + return &EventsUnsubscribeResult{}, nil + }); err != nil { + return nil, err + } + return s, nil +} + +func toolErr(err error) *mcp.CallToolResult { + return &mcp.CallToolResult{IsError: true, Content: []mcp.Content{&mcp.TextContent{Text: fmt.Sprint(err)}}} +} + +// HTTPWithBridge serves MCP with optional events capability advertisement. +func HTTPWithBridge(backend service.Backend, bridge *dotbridge.Service) (http.Handler, error) { + s, err := NewWithBridge(backend, bridge) + if err != nil { + return nil, err + } + base := mcp.NewStreamableHTTPHandler(func(*http.Request) *mcp.Server { return s }, &mcp.StreamableHTTPOptions{Stateless: true, JSONResponse: true}) + return DiscoverEventsWrapper(base), nil +} diff --git a/internal/mcpserver/discover.go b/internal/mcpserver/discover.go new file mode 100644 index 0000000..db493c2 --- /dev/null +++ b/internal/mcpserver/discover.go @@ -0,0 +1,76 @@ +package mcpserver + +import ( + "bytes" + "encoding/json" + "io" + "net/http" +) + +// DiscoverEventsWrapper augments successful 2026-07-28 server/discover responses +// with events capability without replacing SDK dispatch. +func DiscoverEventsWrapper(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + next.ServeHTTP(w, r) + return + } + rec := &responseRecorder{header: make(http.Header)} + next.ServeHTTP(rec, r) + body := rec.body.Bytes() + if rec.status == http.StatusOK && bytes.Contains(body, []byte(`"method":"server/discover"`)) { + body = patchDiscoverEvents(body) + } + for k, vals := range rec.header { + for _, v := range vals { + w.Header().Add(k, v) + } + } + w.WriteHeader(rec.status) + _, _ = io.Copy(w, bytes.NewReader(body)) + }) +} + +type responseRecorder struct { + header http.Header + body bytes.Buffer + status int +} + +func (r *responseRecorder) Header() http.Header { return r.header } +func (r *responseRecorder) Write(b []byte) (int, error) { return r.body.Write(b) } +func (r *responseRecorder) WriteHeader(statusCode int) { r.status = statusCode } + +func patchDiscoverEvents(body []byte) []byte { + var env map[string]json.RawMessage + if json.Unmarshal(body, &env) != nil { + return body + } + result, ok := env["result"] + if !ok { + return body + } + var obj map[string]any + if json.Unmarshal(result, &obj) != nil { + return body + } + caps, ok := obj["capabilities"].(map[string]any) + if !ok { + return body + } + if _, exists := caps["events"]; !exists { + caps["events"] = map[string]any{} + obj["capabilities"] = caps + patched, err := json.Marshal(obj) + if err != nil { + return body + } + env["result"] = patched + out, err := json.Marshal(env) + if err != nil { + return body + } + return out + } + return body +} From 2024033f56a094d4e66cdfff1a175cacf6a3d4c5 Mon Sep 17 00:00:00 2001 From: Pyuyi <204892921+VeraPyuyi@users.noreply.github.com> Date: Mon, 19 Oct 2026 12:54:01 +0800 Subject: [PATCH 09/47] fix(dotbridge): wire queue notices and stabilize runtime tests Carry OccurredAt through queue notices into event delivery, avoid recover deadlock, and add fixture-backed subscription verification for readiness tests. Expose dotevents.NewDelivery for offline fixtures and move MCP bridge tool test out of dotbridge to break import cycles. Verified: go test -run TestBridgeAPIProbeClaimComplete|TestNewWithBridgeAddsTaskTools ./internal/dotbridge ./internal/mcpserver --- internal/dotbridge/adapters.go | 22 +++-- internal/dotbridge/bridge_test.go | 36 +++------ internal/dotbridge/events_fixture_test.go | 84 +++++++++++++++++++ internal/dotbridge/mcp_test.go | 26 ------ internal/dotbridge/probe.go | 6 +- internal/dotbridge/service.go | 46 +++++++---- internal/dotbridge/types.go | 44 +++++----- internal/dotbridge/worker.go | 17 ++-- internal/dotevents/delivery.go | 13 ++- internal/dotqueue/store.go | 9 ++- internal/dotqueue/types.go | 9 ++- internal/mcpserver/bridge_tools_test.go | 98 +++++++++++++++++++++++ 12 files changed, 294 insertions(+), 116 deletions(-) create mode 100644 internal/dotbridge/events_fixture_test.go delete mode 100644 internal/dotbridge/mcp_test.go create mode 100644 internal/mcpserver/bridge_tools_test.go diff --git a/internal/dotbridge/adapters.go b/internal/dotbridge/adapters.go index 62d4e2e..5d17f92 100644 --- a/internal/dotbridge/adapters.go +++ b/internal/dotbridge/adapters.go @@ -37,16 +37,18 @@ type queueAdapter struct{ *dotqueue.Store } type eventsAdapter struct{ *dotevents.Manager } -func wrapQueue(s *dotqueue.Store) QueueAPI { return queueAdapter{s} } +func wrapQueue(s *dotqueue.Store) QueueAPI { return queueAdapter{s} } func wrapEvents(m *dotevents.Manager) EventsAPI { return eventsAdapter{m} } -func (q queueAdapter) Initialize(enabled bool) error { return q.Store.Initialize(enabled) } -func (q queueAdapter) SetEnabled(enabled bool) error { return q.Store.SetEnabled(enabled) } +func (q queueAdapter) Initialize(enabled bool) error { return q.Store.Initialize(enabled) } +func (q queueAdapter) SetEnabled(enabled bool) error { return q.Store.SetEnabled(enabled) } func (q queueAdapter) ConfigStatus() (dotqueue.ConfigStatus, error) { return q.Store.ConfigStatus() } func (q queueAdapter) Submit(clientID string, sub dotqueue.Submission) (dotqueue.Job, error) { return q.Store.Submit(clientID, sub) } -func (q queueAdapter) Get(clientID, jobID string) (dotqueue.Job, error) { return q.Store.Get(clientID, jobID) } +func (q queueAdapter) Get(clientID, jobID string) (dotqueue.Job, error) { + return q.Store.Get(clientID, jobID) +} func (q queueAdapter) ByRequest(clientID, requestID string) (dotqueue.Job, error) { return q.Store.ByRequest(clientID, requestID) } @@ -59,9 +61,13 @@ func (q queueAdapter) Progress(jobID, token, state, summary string) (dotqueue.Jo func (q queueAdapter) Complete(jobID, token string, result dotqueue.Result) (dotqueue.Job, error) { return q.Store.Complete(jobID, token, result) } -func (q queueAdapter) Cancel(clientID, jobID string) (dotqueue.Job, error) { return q.Store.Cancel(clientID, jobID) } +func (q queueAdapter) Cancel(clientID, jobID string) (dotqueue.Job, error) { + return q.Store.Cancel(clientID, jobID) +} func (q queueAdapter) Recover() error { return q.Store.Recover() } -func (q queueAdapter) Pending(queueID string) ([]dotqueue.Notice, error) { return q.Store.Pending(queueID) } +func (q queueAdapter) Pending(queueID string) ([]dotqueue.Notice, error) { + return q.Store.Pending(queueID) +} func (e eventsAdapter) SetEnabled(enabled bool) error { return e.Manager.SetEnabled(enabled) } func (e eventsAdapter) Subscribe(ctx context.Context, principal string, params dotevents.Params) (dotevents.Subscription, error) { @@ -70,7 +76,9 @@ func (e eventsAdapter) Subscribe(ctx context.Context, principal string, params d func (e eventsAdapter) Unsubscribe(ctx context.Context, principal string, params dotevents.Params) error { return e.Manager.Unsubscribe(ctx, principal, params) } -func (e eventsAdapter) List(principal string) []dotevents.Definition { return e.Manager.List(principal) } +func (e eventsAdapter) List(principal string) []dotevents.Definition { + return e.Manager.List(principal) +} func (e eventsAdapter) Deliver(ctx context.Context, principal, queueID string, notice dotevents.Notice) error { return e.Manager.Deliver(ctx, principal, queueID, notice) } diff --git a/internal/dotbridge/bridge_test.go b/internal/dotbridge/bridge_test.go index 9c290df..cd0cf3a 100644 --- a/internal/dotbridge/bridge_test.go +++ b/internal/dotbridge/bridge_test.go @@ -7,10 +7,8 @@ import ( "net/http/httptest" "path/filepath" "testing" - "time" "github.com/dotcontrol/dotcontrol/internal/dotqueue" - "github.com/dotcontrol/dotcontrol/internal/mcpserver" "github.com/dotcontrol/dotcontrol/internal/service" ) @@ -27,15 +25,18 @@ func openTestBridge(t *testing.T) (*Service, string, string) { if err := EnsureStateKey(vault); err != nil { t.Fatal(err) } + delivery, closeFixture := NewTestEventsDelivery(t) + t.Cleanup(closeFixture) ctx, cancel := context.WithCancel(context.Background()) t.Cleanup(cancel) svc, err := Open(ctx, Config{ - Root: filepath.Join(root, "dot-bridge"), - Listen: "127.0.0.1:0", - Vault: vault, - MCPQueue: "queue.a", - Paused: func() bool { return false }, - Revoked: func() bool { return false }, + Root: filepath.Join(root, "dot-bridge"), + Listen: "127.0.0.1:0", + Vault: vault, + MCPQueue: "client.a", + EventsDelivery: delivery, + Paused: func() bool { return false }, + Revoked: func() bool { return false }, }) if err != nil { t.Fatal(err) @@ -48,6 +49,7 @@ func openTestBridge(t *testing.T) (*Service, string, string) { if err != nil { t.Fatal(err) } + VerifyTestSubscription(t, svc, "client.a") var file PairingFile if json.Unmarshal(mustRead(path), &file) != nil { t.Fatal("pairing file invalid") @@ -96,24 +98,6 @@ func TestBridgeAPIProbeClaimComplete(t *testing.T) { } } -func TestMCPAddsBridgeTools(t *testing.T) { - svc, _, _ := openTestBridge(t) - s, err := mcpserver.NewWithBridge(noopBackend{}, svc) - if err != nil { - t.Fatal(err) - } - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - client := mcpTestClient(t, s, ctx) - list, err := client.ListTools(ctx, nil) - if err != nil { - t.Fatal(err) - } - if len(list.Tools) != 17 { - t.Fatalf("tool count=%d", len(list.Tools)) - } -} - func mustProbeAnswer(svc *Service, clientID string) string { svc.mu.Lock() answer := svc.state.Clients[clientID].ProbeAnswer diff --git a/internal/dotbridge/events_fixture_test.go b/internal/dotbridge/events_fixture_test.go new file mode 100644 index 0000000..528a058 --- /dev/null +++ b/internal/dotbridge/events_fixture_test.go @@ -0,0 +1,84 @@ +package dotbridge + +import ( + "context" + "crypto/x509" + "encoding/base64" + "encoding/json" + "io" + "net" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/dotcontrol/dotcontrol/internal/dotevents" +) + +const fixtureCallbackURL = "https://callback.example/hook?opaque=ok" + +func testEventsSecret() string { + return "whsec_" + base64.StdEncoding.EncodeToString([]byte("01234567890123456789012345678901")) +} + +func testEventsParams(queueID string) dotevents.Params { + return dotevents.Params{ + Name: dotevents.EventName, + Arguments: dotevents.Arguments{QueueID: queueID}, + Delivery: dotevents.Callback{ + Mode: "webhook", + URL: fixtureCallbackURL, + Secret: testEventsSecret(), + }, + } +} + +// NewTestEventsDelivery is for offline integration tests only. +func NewTestEventsDelivery(t *testing.T) (*dotevents.Delivery, func()) { + t.Helper() + s := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(io.LimitReader(r.Body, 32*1024)) + if strings.Contains(string(body), "verification") { + var req struct { + Challenge string `json:"challenge"` + } + if json.Unmarshal(body, &req) != nil || req.Challenge == "" { + w.WriteHeader(http.StatusBadRequest) + return + } + _ = json.NewEncoder(w).Encode(map[string]string{"challenge": req.Challenge}) + return + } + w.WriteHeader(http.StatusOK) + })) + pool := x509.NewCertPool() + pool.AddCert(s.Certificate()) + tr := s.Client().Transport.(*http.Transport).Clone() + tr.Proxy = nil + tr.DisableKeepAlives = true + tr.TLSClientConfig.RootCAs = pool + tr.TLSClientConfig.ServerName = "example.com" + tr.DialContext = func(ctx context.Context, network, address string) (net.Conn, error) { + if address != "callback.example:443" { + return nil, &dotevents.Error{Code: "callback"} + } + return (&net.Dialer{Timeout: time.Second}).DialContext(ctx, network, s.Listener.Addr().String()) + } + return dotevents.NewDelivery(&http.Client{ + Timeout: 10 * time.Second, + Transport: tr, + CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }, + }), s.Close +} + +// VerifyTestSubscription exercises callback verification for readiness tests. +func VerifyTestSubscription(t *testing.T, svc *Service, queueID string) { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + sub, err := svc.EventsSubscribe(ctx, queueID, testEventsParams(queueID)) + if err != nil || !sub.CallbackVerified { + t.Fatalf("subscribe verify failed: %v %#v", err, sub) + } +} diff --git a/internal/dotbridge/mcp_test.go b/internal/dotbridge/mcp_test.go deleted file mode 100644 index 2340257..0000000 --- a/internal/dotbridge/mcp_test.go +++ /dev/null @@ -1,26 +0,0 @@ -package dotbridge - -import ( - "context" - "testing" - "time" - - "github.com/modelcontextprotocol/go-sdk/mcp" -) - -func mcpTestClient(t *testing.T, s *mcp.Server, ctx context.Context) *mcp.ClientSession { - t.Helper() - client := mcp.NewClient(&mcp.Implementation{Name: "bridge-test", Version: "1"}, nil) - st, mem := mcp.NewInMemoryTransports() - sess, err := s.Connect(ctx, st, nil) - if err != nil { - t.Fatal(err) - } - t.Cleanup(func() { sess.Close() }) - cs, err := client.Connect(ctx, mem, nil) - if err != nil { - t.Fatal(err) - } - t.Cleanup(func() { cs.Close() }) - return cs -} diff --git a/internal/dotbridge/probe.go b/internal/dotbridge/probe.go index 4880769..94f5403 100644 --- a/internal/dotbridge/probe.go +++ b/internal/dotbridge/probe.go @@ -11,9 +11,9 @@ import ( ) type ProbeResult struct { - JobID string `json:"job_id"` - RequestID string `json:"request_id"` - QueueID string `json:"queue_id"` + JobID string `json:"job_id"` + RequestID string `json:"request_id"` + QueueID string `json:"queue_id"` Steps []string `json:"steps"` } diff --git a/internal/dotbridge/service.go b/internal/dotbridge/service.go index 05613a1..46cf25e 100644 --- a/internal/dotbridge/service.go +++ b/internal/dotbridge/service.go @@ -34,13 +34,14 @@ type Service struct { } type Config struct { - Root string - Listen string - Vault vault.Store - MCPQueue string - MCPEpoch uint64 - Paused func() bool - Revoked func() bool + Root string + Listen string + Vault vault.Store + MCPQueue string + MCPEpoch uint64 + Paused func() bool + Revoked func() bool + EventsDelivery *dotevents.Delivery } func Open(ctx context.Context, cfg Config) (*Service, error) { @@ -61,7 +62,11 @@ func Open(ctx context.Context, cfg Config) (*Service, error) { if err != nil { return nil, err } - eventsMgr, err := dotevents.Open(eventsDir, key, &dotevents.Delivery{}) + delivery := cfg.EventsDelivery + if delivery == nil { + delivery = &dotevents.Delivery{} + } + eventsMgr, err := dotevents.Open(eventsDir, key, delivery) if err != nil { return nil, err } @@ -84,9 +89,21 @@ func Open(ctx context.Context, cfg Config) (*Service, error) { cancel() return nil, err } - if err := s.recoverOnce(); err != nil { - cancel() - return nil, err + if s.state.Enabled { + s.mu.Lock() + defer s.mu.Unlock() + if err := s.queue.SetEnabled(true); err != nil { + cancel() + return nil, err + } + if err := s.events.SetEnabled(true); err != nil { + cancel() + return nil, err + } + if err := s.recoverOnce(); err != nil { + cancel() + return nil, err + } } s.worker = newWorker(s) s.worker.Start() @@ -104,8 +121,6 @@ func (s *Service) Close() error { } func (s *Service) recoverOnce() error { - s.mu.Lock() - defer s.mu.Unlock() if s.recovered { return nil } @@ -141,6 +156,9 @@ func (s *Service) Enable() error { if err := s.events.SetEnabled(true); err != nil { return err } + if err := s.recoverOnce(); err != nil { + return err + } s.state.Enabled = true return s.saveState() } @@ -348,7 +366,7 @@ func (s *Service) submit(client ClientPolicy, req SubmitRequest) (JobDTO, int, e if err := s.gate(); err != nil { return JobDTO{}, 0, err } - status := s.statusDTO(policy) + status := s.statusDTO(client) if !status.Ready && req.Mode != dotqueue.ModeAnswer { return JobDTO{}, 0, errors.New("probe_required") } diff --git a/internal/dotbridge/types.go b/internal/dotbridge/types.go index fc524d1..d4ecf5c 100644 --- a/internal/dotbridge/types.go +++ b/internal/dotbridge/types.go @@ -24,26 +24,26 @@ type ClientPolicy struct { } type CapPolicy struct { - Text bool `json:"text"` + Text bool `json:"text"` Tasks bool `json:"tasks"` } type PersistedState struct { - Version int `json:"version"` - Enabled bool `json:"enabled"` + Version int `json:"version"` + Enabled bool `json:"enabled"` Clients map[string]ClientPolicy `json:"clients"` } type StatusDTO struct { - Version int `json:"version"` - Enabled bool `json:"enabled"` - Paired bool `json:"paired"` - Ready bool `json:"ready"` - Code string `json:"code"` - VerifiedAt *time.Time `json:"verified_at,omitempty"` - ExpiresAt *time.Time `json:"expires_at,omitempty"` - Subscription SubscriptionDTO `json:"subscription"` - Capabilities CapPolicy `json:"capabilities"` + Version int `json:"version"` + Enabled bool `json:"enabled"` + Paired bool `json:"paired"` + Ready bool `json:"ready"` + Code string `json:"code"` + VerifiedAt *time.Time `json:"verified_at,omitempty"` + ExpiresAt *time.Time `json:"expires_at,omitempty"` + Subscription SubscriptionDTO `json:"subscription"` + Capabilities CapPolicy `json:"capabilities"` } type SubscriptionDTO struct { @@ -74,11 +74,11 @@ type ResultDTO struct { } type SubmitRequest struct { - RequestID string `json:"request_id"` - SessionID string `json:"session_id"` - Mode string `json:"mode"` - Messages []MessageDTO `json:"messages,omitempty"` - Task string `json:"task,omitempty"` + RequestID string `json:"request_id"` + SessionID string `json:"session_id"` + Mode string `json:"mode"` + Messages []MessageDTO `json:"messages,omitempty"` + Task string `json:"task,omitempty"` } type MessageDTO struct { @@ -96,10 +96,10 @@ type ErrorBody struct { } type PairingFile struct { - Version int `json:"version"` - ClientID string `json:"client_id"` - QueueID string `json:"queue_id"` - Token string `json:"token"` - Listen string `json:"listen"` + Version int `json:"version"` + ClientID string `json:"client_id"` + QueueID string `json:"queue_id"` + Token string `json:"token"` + Listen string `json:"listen"` CreatedAt time.Time `json:"created_at"` } diff --git a/internal/dotbridge/worker.go b/internal/dotbridge/worker.go index bb47027..8140dd7 100644 --- a/internal/dotbridge/worker.go +++ b/internal/dotbridge/worker.go @@ -10,9 +10,9 @@ import ( ) type worker struct { - svc *Service - done chan struct{} - wg sync.WaitGroup + svc *Service + done chan struct{} + wg sync.WaitGroup } func newWorker(s *Service) *worker { @@ -68,10 +68,11 @@ func (w *worker) tick() { } ctx, cancel := context.WithTimeout(w.svc.ctx, 5*time.Second) err := w.svc.events.Deliver(ctx, queueID, queueID, dotevents.Notice{ - EventID: notice.EventID, - QueueID: notice.QueueID, - JobID: notice.JobID, - Mode: notice.Mode, + EventID: notice.EventID, + QueueID: notice.QueueID, + JobID: notice.JobID, + Mode: notice.Mode, + OccurredAt: notice.OccurredAt, }) cancel() if err != nil { @@ -90,7 +91,7 @@ func (s *Service) claimTask(queueID, jobID, workerID string) (dotqueue.Claim, er return dotqueue.Claim{}, err } if queueID != authorized { - return dotqueue.Claim{}, dotqueue.Error{Code: "unknown"} + return dotqueue.Claim{}, &dotqueue.Error{Code: "unknown"} } return s.queue.Claim(queueID, jobID, workerID) } diff --git a/internal/dotevents/delivery.go b/internal/dotevents/delivery.go index 4ade029..e4460d3 100644 --- a/internal/dotevents/delivery.go +++ b/internal/dotevents/delivery.go @@ -28,10 +28,19 @@ const ( var retrySchedule = []time.Duration{time.Second, 2 * time.Second, 4 * time.Second} -// Delivery's zero value uses the public HTTPS transport. Custom transports are -// deliberately private; offline TLS fixtures construct them inside package tests. +// Delivery's zero value uses the public HTTPS transport. Test fixtures may +// supply a custom HTTP client through NewDelivery. type Delivery struct{ client *http.Client } +// NewDelivery constructs a delivery transport with an explicit HTTP client. +// Production callers should pass nil to use the public HTTPS transport. +func NewDelivery(client *http.Client) *Delivery { + if client == nil { + return &Delivery{} + } + return &Delivery{client: client} +} + func (d *Delivery) httpClient() *http.Client { if d != nil && d.client != nil { return d.client diff --git a/internal/dotqueue/store.go b/internal/dotqueue/store.go index 8c400fb..666d7b8 100644 --- a/internal/dotqueue/store.go +++ b/internal/dotqueue/store.go @@ -375,10 +375,11 @@ func (s *Store) Pending(queueID string) ([]Notice, error) { continue } out = append(out, Notice{ - EventID: rec.EventID, - QueueID: rec.QueueID, - JobID: rec.ID, - Mode: rec.Mode, + EventID: rec.EventID, + QueueID: rec.QueueID, + JobID: rec.ID, + Mode: rec.Mode, + OccurredAt: rec.CreatedAt, }) } return nil diff --git a/internal/dotqueue/types.go b/internal/dotqueue/types.go index 822a0b9..6fd31b6 100644 --- a/internal/dotqueue/types.go +++ b/internal/dotqueue/types.go @@ -66,8 +66,9 @@ type Artifact struct { } type Notice struct { - EventID string `json:"event_id"` - QueueID string `json:"queue_id"` - JobID string `json:"job_id"` - Mode string `json:"mode"` + EventID string `json:"event_id"` + QueueID string `json:"queue_id"` + JobID string `json:"job_id"` + Mode string `json:"mode"` + OccurredAt time.Time `json:"occurred_at"` } diff --git a/internal/mcpserver/bridge_tools_test.go b/internal/mcpserver/bridge_tools_test.go new file mode 100644 index 0000000..a91de71 --- /dev/null +++ b/internal/mcpserver/bridge_tools_test.go @@ -0,0 +1,98 @@ +package mcpserver + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "testing" + "time" + + "github.com/dotcontrol/dotcontrol/internal/dotbridge" + "github.com/dotcontrol/dotcontrol/internal/service" + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +type bridgeNoopBackend struct{} + +func (bridgeNoopBackend) Call(context.Context, string, service.Request) (any, error) { + return map[string]any{"ok": true}, nil +} + +func openBridgeForMCPTest(t *testing.T) *dotbridge.Service { + t.Helper() + root := t.TempDir() + vault := dotbridge.NewFakeVault() + if err := dotbridge.EnsureStateKey(vault); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + svc, err := dotbridge.Open(ctx, dotbridge.Config{ + Root: filepath.Join(root, "dot-bridge"), + Listen: "127.0.0.1:0", + Vault: vault, + MCPQueue: "client.a", + Paused: func() bool { return false }, + Revoked: func() bool { return false }, + }) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = svc.Close() }) + if err := svc.Enable(); err != nil { + t.Fatal(err) + } + _, path, err := svc.Pair("client.a") + if err != nil { + t.Fatal(err) + } + var file dotbridge.PairingFile + if json.Unmarshal(mustReadFile(path), &file) != nil { + t.Fatal("pairing file invalid") + } + return svc +} + +func mustReadFile(path string) []byte { + data, err := os.ReadFile(path) + if err != nil { + panic(err) + } + return data +} + +func mcpBridgeTestClient(t *testing.T, s *mcp.Server, ctx context.Context) *mcp.ClientSession { + t.Helper() + client := mcp.NewClient(&mcp.Implementation{Name: "bridge-test", Version: "1"}, nil) + st, mem := mcp.NewInMemoryTransports() + sess, err := s.Connect(ctx, st, nil) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = sess.Close() }) + cs, err := client.Connect(ctx, mem, nil) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = cs.Close() }) + return cs +} + +func TestNewWithBridgeAddsTaskTools(t *testing.T) { + svc := openBridgeForMCPTest(t) + s, err := NewWithBridge(bridgeNoopBackend{}, svc) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + client := mcpBridgeTestClient(t, s, ctx) + list, err := client.ListTools(ctx, nil) + if err != nil { + t.Fatal(err) + } + if len(list.Tools) != 17 { + t.Fatalf("tool count=%d want 17", len(list.Tools)) + } +} From 5628c72e552dd555c0997653ad9b327f0bf0aff8 Mon Sep 17 00:00:00 2001 From: Pyuyi <204892921+VeraPyuyi@users.noreply.github.com> Date: Mon, 19 Oct 2026 13:54:07 +0800 Subject: [PATCH 10/47] fix(dotbridge): authorize paired queues and repair HTTP discovery Fix C1/C2 and I1-I6: preserve implicit HTTP 200, narrowly patch modern SDK discovery, bind the trusted consumer to paired queues and claims, recheck gates after I/O, synchronize enabled reads, preserve concurrent revoke, repair cancel routing and degrade optional bridge failures. Flatten Events params and bound readiness to subscription generation/expiry; hash nonce expectation and refuse unverified task mode. Behavioral RED: reproduced real SDK HTTP panic/EOF, cancel 404, stale revoke, post-I/O pause and valid JSON rejection. GREEN: targeted dotbridge/mcpserver/dotqueue/dotevents tests; CGO MinGW race dotbridge/mcpserver; optional runtime vault test; go build ./...; targeted go vet. Full fixture covers authenticated MCP HTTP, TLS callback verification/notice, generated paired queue, probe completion/readiness and client answer result read. --- cmd/dotcontrol/dot_bridge.go | 8 + cmd/dotcontrol/dot_bridge_test.go | 23 +++ cmd/dotcontrol/runtime.go | 21 +-- docs/DOT_BRIDGE.md | 24 ++- internal/dotbridge/adapters.go | 6 +- internal/dotbridge/api.go | 8 +- internal/dotbridge/authorization.go | 54 +++++++ internal/dotbridge/bridge_test.go | 10 +- internal/dotbridge/fakevault.go | 13 +- internal/dotbridge/gates_test.go | 193 +++++++++++++++++++++++ internal/dotbridge/http_flow_test.go | 171 +++++++++++++++++++++ internal/dotbridge/pairing.go | 10 +- internal/dotbridge/probe.go | 18 ++- internal/dotbridge/review_test.go | 78 ++++++++++ internal/dotbridge/service.go | 205 ++++++++++++++++--------- internal/dotbridge/types.go | 10 +- internal/dotbridge/worker.go | 108 ++++++++----- internal/mcpserver/bridge.go | 31 ++-- internal/mcpserver/bridge_http_test.go | 55 +++++++ internal/mcpserver/discover.go | 72 +++++---- internal/mcpserver/discover_test.go | 23 +++ 21 files changed, 914 insertions(+), 227 deletions(-) create mode 100644 cmd/dotcontrol/dot_bridge_test.go create mode 100644 internal/dotbridge/authorization.go create mode 100644 internal/dotbridge/gates_test.go create mode 100644 internal/dotbridge/http_flow_test.go create mode 100644 internal/dotbridge/review_test.go create mode 100644 internal/mcpserver/bridge_http_test.go create mode 100644 internal/mcpserver/discover_test.go diff --git a/cmd/dotcontrol/dot_bridge.go b/cmd/dotcontrol/dot_bridge.go index f873cb3..c95d8bd 100644 --- a/cmd/dotcontrol/dot_bridge.go +++ b/cmd/dotcontrol/dot_bridge.go @@ -132,3 +132,11 @@ func loadPairingToken(dir, clientID string) (string, error) { } return file.Token, nil } + +// Optional bridge failures preserve the existing runtime and its default catalog. +func openRuntimeDotBridge(ctx context.Context, dir string, c config.Config, v vault.Store, paused, revoked func() bool) (*dotbridge.Service, error) { + if e := dotbridge.EnsureStateKey(v); e != nil { + return nil, e + } + return dotbridge.Open(ctx, dotbridge.Config{Root: filepath.Join(dir, "dot-bridge"), Listen: c.Listen, Vault: v, MCPQueue: c.DeviceID, Paused: paused, Revoked: revoked}) +} diff --git a/cmd/dotcontrol/dot_bridge_test.go b/cmd/dotcontrol/dot_bridge_test.go new file mode 100644 index 0000000..077180c --- /dev/null +++ b/cmd/dotcontrol/dot_bridge_test.go @@ -0,0 +1,23 @@ +package main + +import ( + "context" + "errors" + "testing" + + "github.com/dotcontrol/dotcontrol/internal/config" +) + +type unavailableBridgeVault struct{ writes int } + +func (v *unavailableBridgeVault) Get(string) (string, error) { + return "", errors.New("vault unavailable") +} +func (v *unavailableBridgeVault) Set(string, string) error { v.writes++; return nil } +func TestOptionalBridgeUnavailableDoesNotReplaceKey(t *testing.T) { + v := &unavailableBridgeVault{} + s, e := openRuntimeDotBridge(context.Background(), t.TempDir(), config.Config{Listen: "127.0.0.1:9317"}, v, nil, nil) + if e == nil || s != nil || v.writes != 0 { + t.Fatalf("service=%v err=%v writes=%d", s, e, v.writes) + } +} diff --git a/cmd/dotcontrol/runtime.go b/cmd/dotcontrol/runtime.go index 25dd0e2..b50d219 100644 --- a/cmd/dotcontrol/runtime.go +++ b/cmd/dotcontrol/runtime.go @@ -182,26 +182,7 @@ func serve(ctx context.Context, dir string, c config.Config, v vault.Vault, args } owner := &runtimeOwner{ctx: ctx, stop: cancel, dir: dir, cfg: c, vault: v, eng: eng, monitor: monitor, sup: &tunnel.Supervisor{}, revoked: revErr == nil, noTunnel: *noTunnel, repair: make(chan struct{}, 1)} if c.DotBridgeEnabled { - if err := dotbridge.EnsureStateKey(v); err != nil { - return err - } - dotBridge, err := dotbridge.Open(ctx, dotbridge.Config{ - Root: filepath.Join(dir, "dot-bridge"), - Listen: c.Listen, - Vault: v, - MCPQueue: c.DeviceID, - Paused: func() bool { return eng.Device().Paused }, - Revoked: func() bool { - owner.mu.Lock() - rev := owner.revoked - owner.mu.Unlock() - return rev - }, - }) - if err != nil { - return err - } - owner.dotBridge = dotBridge + owner.dotBridge, _ = openRuntimeDotBridge(ctx, dir, c, v, func() bool { return eng.Device().Paused }, func() bool { owner.mu.Lock(); defer owner.mu.Unlock(); return owner.revoked }) } if owner.revoked { _ = eng.Pause(true) diff --git a/docs/DOT_BRIDGE.md b/docs/DOT_BRIDGE.md index 85fcdee..4258b3c 100644 --- a/docs/DOT_BRIDGE.md +++ b/docs/DOT_BRIDGE.md @@ -1,19 +1,17 @@ # Dot bridge -The Dot bridge is opt-in via `dot_bridge_enabled` in local config. It does not change the default 13-tool MCP catalog unless the runtime is started with bridge support enabled. +The bridge is optional. Set `dot_bridge_enabled` in the installed local configuration and restart the runtime, then run `dotcontrol bridge enable`. The default installation retains 13 MCP tools. With the bridge available, MCP adds four task tools and three Events methods. A missing or unavailable vault key, or a bridge open failure, leaves the regular runtime and its 13 tools available; bridge admin operations report `bridge_unavailable`. -## Operator flow +Run `dotcontrol bridge pair` to create a separate client credential and queue. The command prints a pairing file path and public IDs. Import the file locally into the client, then remove it when import is complete. Pairing locally authorizes the runtime's configured Dot consumer to act on that queue. Pairing grants no desktop, history, or terminal permission. -1. Enable bridge support in config and start `dotcontrol serve`. -2. Run `dotcontrol bridge enable`. -3. Run `dotcontrol bridge pair` and import the secure pairing file into Magpie. -4. Connect the private Dot plugin, refresh MCP metadata, and subscribe to `dot.task.available`. -5. Run `dotcontrol bridge probe ` and complete the probe job through MCP claim/complete. -6. Poll `GET /dot-bridge/v1/status` with the paired Bearer token. +Connect the private Dot plugin, refresh its metadata, and subscribe to `dot.task.available` using the returned queue ID. Events parameters are flat: `name`, `arguments`, `delivery`, and optional `cursor`/`ttlMs`. The SDK handles the Events RPC methods and protocol metadata checks. Only a successful modern `2026-07-28` `server/discover` response advertises `capabilities.events`. -## Security notes +Run `dotcontrol bridge probe ` explicitly to enqueue a nonce challenge. Through MCP, claim that job with `dot_task_claim`, then complete it with the exact requested nonce. The service checks the claim token, queue authorization, current verified callback subscription generation, and completed answer state. Expected nonce text is hashed in bridge policy; the challenge prompt remains in the encrypted queue. No manual readiness setter exists. -- Loopback client API uses Bearer tokens only; queue scope is resolved from the paired token. -- MCP task tools bind to the configured bridge queue, not arbitrary client input. -- Readiness is derived from a real probe claim/completion lifecycle; there is no manual readiness setter. -- The bridge cannot cryptographically prove that Dot itself completed the probe versus another authorized MCP caller using the same tunnel credential. +The paired Bearer credential accesses `/dot-bridge/v1/status`, `/jobs`, `/jobs/{id}`, `/requests/{request_id}`, and `/jobs/{id}/cancel` on the numeric loopback listener. No client-supplied identity selects a queue. Status exposes no prompt or credential. A successful text probe enables text readiness for at most ten minutes, bounded by subscription expiry. Pause, revoke, disable, renewal, and unsubscribe invalidate or suppress readiness. Runtime restart requires a fresh subscription and probe; live claim bindings are intentionally not restored. + +Regular submission requires current readiness. Task execution remains unavailable: `capabilities.tasks` is false and task submissions return `task_verification_required`. A text ping establishes no evidence about cloud task execution. An authorized task verification policy and work test are required before enabling that capability. + +All MCP callers sharing the authenticated installation hop act as the same configured consumer. Queue IDs and worker IDs provide no authentication by themselves. A worker ID supplies retry identity and is combined with the trusted consumer identity. This proves a round trip through the configured source, not cryptographic model or personal identity. Fixture tests provide protocol evidence only, with no real account acceptance. + +Operations recheck pause, revocation, enabled state, and queue policy after queue or callback I/O. A concurrent gate change can suppress a success response even when a durable write has happened. Resolve uncertain submission through the original request ID; never replay under a new ID. Cancellation records intent and cannot prove that cloud work stopped. diff --git a/internal/dotbridge/adapters.go b/internal/dotbridge/adapters.go index 5d17f92..d7158ad 100644 --- a/internal/dotbridge/adapters.go +++ b/internal/dotbridge/adapters.go @@ -11,7 +11,6 @@ import ( type QueueAPI interface { Initialize(enabled bool) error SetEnabled(enabled bool) error - ConfigStatus() (dotqueue.ConfigStatus, error) Submit(clientID string, sub dotqueue.Submission) (dotqueue.Job, error) Get(clientID, jobID string) (dotqueue.Job, error) ByRequest(clientID, requestID string) (dotqueue.Job, error) @@ -40,9 +39,8 @@ type eventsAdapter struct{ *dotevents.Manager } func wrapQueue(s *dotqueue.Store) QueueAPI { return queueAdapter{s} } func wrapEvents(m *dotevents.Manager) EventsAPI { return eventsAdapter{m} } -func (q queueAdapter) Initialize(enabled bool) error { return q.Store.Initialize(enabled) } -func (q queueAdapter) SetEnabled(enabled bool) error { return q.Store.SetEnabled(enabled) } -func (q queueAdapter) ConfigStatus() (dotqueue.ConfigStatus, error) { return q.Store.ConfigStatus() } +func (q queueAdapter) Initialize(enabled bool) error { return q.Store.Initialize(enabled) } +func (q queueAdapter) SetEnabled(enabled bool) error { return q.Store.SetEnabled(enabled) } func (q queueAdapter) Submit(clientID string, sub dotqueue.Submission) (dotqueue.Job, error) { return q.Store.Submit(clientID, sub) } diff --git a/internal/dotbridge/api.go b/internal/dotbridge/api.go index 2450ac7..dc5a5ff 100644 --- a/internal/dotbridge/api.go +++ b/internal/dotbridge/api.go @@ -86,11 +86,15 @@ func (s *Service) handleJobs(w http.ResponseWriter, r *http.Request) { func (s *Service) handleJobByID(w http.ResponseWriter, r *http.Request) { client := clientFrom(r.Context()) jobID := strings.TrimPrefix(r.URL.Path, "/dot-bridge/v1/jobs/") + cancelRoute := strings.HasSuffix(jobID, "/cancel") + if cancelRoute { + jobID = strings.TrimSuffix(jobID, "/cancel") + } if jobID == "" || strings.Contains(jobID, "/") { writeError(w, http.StatusNotFound, "notfound", "") return } - if strings.HasSuffix(r.URL.Path, "/cancel") { + if cancelRoute { if r.Method != http.MethodPost { writeError(w, http.StatusMethodNotAllowed, "method_rejected", "") return @@ -142,7 +146,7 @@ func decodeSubmit(body io.Reader) (SubmitRequest, error) { if dec.Decode(&req) != nil { return SubmitRequest{}, errors.New("invalid_request") } - if dec.Decode(&struct{}{}) != nil { + if dec.Decode(&struct{}{}) != io.EOF { return SubmitRequest{}, errors.New("invalid_request") } return req, nil diff --git a/internal/dotbridge/authorization.go b/internal/dotbridge/authorization.go new file mode 100644 index 0000000..16eb51a --- /dev/null +++ b/internal/dotbridge/authorization.go @@ -0,0 +1,54 @@ +package dotbridge + +import ( + "crypto/sha256" + "encoding/hex" + "errors" + "time" +) + +type subscriptionProof struct { + Expires time.Time + Generation uint64 +} +type claimBinding struct { + QueueID, Token string + Generation uint64 +} + +// The runtime's authenticated single consumer may act only on locally paired queues. +func (s *Service) authorizeQueue(queue string) error { + if e := s.gate(); e != nil { + return e + } + s.mu.Lock() + defer s.mu.Unlock() + if s.mcpQueue == "" { + return errors.New("mcp_consumer_unconfigured") + } + p, ok := s.state.Clients[queue] + if !ok || p.Revoked || p.QueueID != queue { + return errors.New("queue_unauthorized") + } + return nil +} +func (s *Service) clientGate(id string) error { return s.authorizeQueue(id) } +func (s *Service) binding(job, token string) (claimBinding, error) { + s.mu.Lock() + b, ok := s.claims[job] + s.mu.Unlock() + if !ok || !tokenMatches(hashToken(b.Token), token) { + return b, errors.New("claim_unauthorized") + } + return b, s.authorizeQueue(b.QueueID) +} +func (s *Service) activeSubscription(queue string) (subscriptionProof, bool) { + s.mu.Lock() + sub, ok := s.subscriptions[queue] + s.mu.Unlock() + return sub, ok && time.Now().Before(sub.Expires) +} +func consumerWorker(consumer, worker string) string { + h := sha256.Sum256([]byte(consumer + "\x00" + worker)) + return "consumer." + hex.EncodeToString(h[:]) +} diff --git a/internal/dotbridge/bridge_test.go b/internal/dotbridge/bridge_test.go index cd0cf3a..e2f679a 100644 --- a/internal/dotbridge/bridge_test.go +++ b/internal/dotbridge/bridge_test.go @@ -6,6 +6,7 @@ import ( "net/http" "net/http/httptest" "path/filepath" + "strings" "testing" "github.com/dotcontrol/dotcontrol/internal/dotqueue" @@ -79,7 +80,7 @@ func TestBridgeAPIProbeClaimComplete(t *testing.T) { if err != nil { t.Fatal(err) } - job, err := svc.CompleteTask(probe.JobID, claim.Token, dotqueue.Result{Text: mustProbeAnswer(svc, clientID)}) + job, err := svc.CompleteTask(probe.JobID, claim.Token, dotqueue.Result{Text: strings.TrimPrefix(claim.Submission.Messages[0].Content, "Reply with exactly this nonce: ")}) if err != nil || job.State != dotqueue.StateCompleted { t.Fatalf("complete: %v %#v", err, job) } @@ -97,10 +98,3 @@ func TestBridgeAPIProbeClaimComplete(t *testing.T) { t.Fatalf("status code %d", res.Code) } } - -func mustProbeAnswer(svc *Service, clientID string) string { - svc.mu.Lock() - answer := svc.state.Clients[clientID].ProbeAnswer - svc.mu.Unlock() - return answer -} diff --git a/internal/dotbridge/fakevault.go b/internal/dotbridge/fakevault.go index c8db140..cc03598 100644 --- a/internal/dotbridge/fakevault.go +++ b/internal/dotbridge/fakevault.go @@ -1,6 +1,9 @@ package dotbridge -import "sync" +import ( + "github.com/zalando/go-keyring" + "sync" +) type FakeVault struct { mu sync.Mutex @@ -17,7 +20,7 @@ func (f *FakeVault) Get(name string) (string, error) { if v, ok := f.data[name]; ok { return v, nil } - return "", errMissing + return "", keyring.ErrNotFound } func (f *FakeVault) Set(name, value string) error { @@ -26,9 +29,3 @@ func (f *FakeVault) Set(name, value string) error { f.data[name] = value return nil } - -var errMissing = &vaultMissing{} - -type vaultMissing struct{} - -func (v *vaultMissing) Error() string { return "not_found" } diff --git a/internal/dotbridge/gates_test.go b/internal/dotbridge/gates_test.go new file mode 100644 index 0000000..1a71898 --- /dev/null +++ b/internal/dotbridge/gates_test.go @@ -0,0 +1,193 @@ +package dotbridge + +import ( + "context" + "errors" + "sync" + "testing" + "time" + + "github.com/dotcontrol/dotcontrol/internal/dotevents" + "github.com/dotcontrol/dotcontrol/internal/dotqueue" +) + +type afterQueue struct { + QueueAPI + after func() +} + +func (q afterQueue) ByRequest(a, b string) (dotqueue.Job, error) { + j, e := q.QueueAPI.ByRequest(a, b) + q.after() + return j, e +} +func (q afterQueue) Cancel(a, b string) (dotqueue.Job, error) { + j, e := q.QueueAPI.Cancel(a, b) + q.after() + return j, e +} +func (q afterQueue) Claim(a, b, c string) (dotqueue.Claim, error) { + j, e := q.QueueAPI.Claim(a, b, c) + q.after() + return j, e +} +func (q afterQueue) Progress(a, b, c, d string) (dotqueue.Job, error) { + j, e := q.QueueAPI.Progress(a, b, c, d) + q.after() + return j, e +} +func (q afterQueue) Complete(a, b string, r dotqueue.Result) (dotqueue.Job, error) { + j, e := q.QueueAPI.Complete(a, b, r) + q.after() + return j, e +} + +type afterEvents struct { + EventsAPI + after func() +} + +func (e afterEvents) Subscribe(ctx context.Context, p string, params dotevents.Params) (dotevents.Subscription, error) { + s, err := e.EventsAPI.Subscribe(ctx, p, params) + e.after() + return s, err +} +func (e afterEvents) Unsubscribe(ctx context.Context, p string, params dotevents.Params) error { + err := e.EventsAPI.Unsubscribe(ctx, p, params) + e.after() + return err +} + +func TestPostIORevocationGates(t *testing.T) { + for _, op := range []string{"request", "cancel", "claim", "progress", "complete", "subscribe", "unsubscribe"} { + t.Run(op, func(t *testing.T) { + s, _, id := openTestBridge(t) + probe, e := s.StartProbe(id) + if e != nil { + t.Fatal(e) + } + claim, e := s.ClaimTask(id, probe.JobID, "retry") + if e != nil { + t.Fatal(e) + } + revoke := func() { + if e := s.RevokeClient(id); e != nil { + t.Fatal(e) + } + } + s.queue = afterQueue{QueueAPI: s.queue, after: revoke} + s.events = afterEvents{EventsAPI: s.events, after: revoke} + switch op { + case "request": + _, e = s.byRequest(ClientPolicy{ClientID: id}, probe.RequestID) + case "cancel": + _, e = s.cancelJob(ClientPolicy{ClientID: id}, probe.JobID) + case "claim": + _, e = s.ClaimTask(id, probe.JobID, "retry") + case "progress": + _, e = s.ProgressTask(probe.JobID, claim.Token, dotqueue.StateRunning, "running") + case "complete": + _, e = s.CompleteTask(probe.JobID, claim.Token, dotqueue.Result{Text: "reported"}) + case "subscribe": + _, e = s.EventsSubscribe(context.Background(), id, testEventsParams(id)) + case "unsubscribe": + e = s.EventsUnsubscribe(context.Background(), id, testEventsParams(id)) + } + if e == nil { + t.Fatal("success after concurrent revocation") + } + }) + } +} +func TestReadinessSubscriptionGenerationAndExpiry(t *testing.T) { + s, _, id := openTestBridge(t) + p, e := s.StartProbe(id) + if e != nil { + t.Fatal(e) + } + c, e := s.ClaimTask(id, p.JobID, "attempt") + if e != nil { + t.Fatal(e) + } + answer := c.Submission.Messages[0].Content[len("Reply with exactly this nonce: "):] + // Renewing a subscription invalidates claims made under its previous generation. + VerifyTestSubscription(t, s, id) + if _, e = s.ClaimTask(id, p.JobID, "attempt"); e != nil { + t.Fatal(e) + } + if _, e = s.CompleteTask(p.JobID, c.Token, dotqueue.Result{Text: answer}); e != nil { + t.Fatal(e) + } + status, _ := s.StatusForClient(id) + if status.Ready { + t.Fatal("old subscription claim promoted readiness") + } + p, e = s.StartProbe(id) + if e != nil { + t.Fatal(e) + } + c, e = s.ClaimTask(id, p.JobID, "attempt2") + if e != nil { + t.Fatal(e) + } + answer = c.Submission.Messages[0].Content[len("Reply with exactly this nonce: "):] + s.mu.Lock() + sub := s.subscriptions[id] + sub.Expires = time.Now().Add(time.Minute) + s.subscriptions[id] = sub + s.mu.Unlock() + if _, e = s.CompleteTask(p.JobID, c.Token, dotqueue.Result{Text: answer}); e != nil { + t.Fatal(e) + } + status, _ = s.StatusForClient(id) + if !status.Ready || status.ExpiresAt.After(sub.Expires) { + t.Fatalf("expiry: %#v", status) + } +} +func TestGateEnabledConcurrent(t *testing.T) { + s, _, _ := openTestBridge(t) + var wg sync.WaitGroup + wg.Add(2) + go func() { + defer wg.Done() + for i := 0; i < 100; i++ { + _ = s.gate() + } + }() + go func() { + defer wg.Done() + for i := 0; i < 100; i++ { + s.mu.Lock() + s.state.Enabled = !s.state.Enabled + s.mu.Unlock() + } + }() + wg.Wait() +} +func TestForeignQueueAndClaimRefused(t *testing.T) { + s, _, id := openTestBridge(t) + p, e := s.StartProbe(id) + if e != nil { + t.Fatal(e) + } + if _, e = s.ClaimTask("foreign", p.JobID, "attempt"); e == nil { + t.Fatal("foreign queue") + } + if _, e = s.TaskStatus(id, p.JobID, "forged"); e == nil { + t.Fatal("forged token") + } + if e = s.EventsUnsubscribe(context.Background(), id, testEventsParams("foreign")); e == nil { + t.Fatal("foreign events") + } +} + +type brokenVault struct{ writes int } + +func (v *brokenVault) Get(string) (string, error) { return "", errors.New("vault unavailable") } +func (v *brokenVault) Set(string, string) error { v.writes++; return nil } +func TestUnavailableVaultNotOverwritten(t *testing.T) { + v := &brokenVault{} + if EnsureStateKey(v) == nil || v.writes != 0 { + t.Fatal("overwrote unavailable vault") + } +} diff --git a/internal/dotbridge/http_flow_test.go b/internal/dotbridge/http_flow_test.go new file mode 100644 index 0000000..ffaf36a --- /dev/null +++ b/internal/dotbridge/http_flow_test.go @@ -0,0 +1,171 @@ +package dotbridge_test + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + "io" + "net" + "net/http" + "net/http/httptest" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/dotcontrol/dotcontrol/internal/dotbridge" + "github.com/dotcontrol/dotcontrol/internal/dotevents" + "github.com/dotcontrol/dotcontrol/internal/dotqueue" + "github.com/dotcontrol/dotcontrol/internal/localapi" + "github.com/dotcontrol/dotcontrol/internal/mcpserver" + "github.com/dotcontrol/dotcontrol/internal/service" +) + +type backend struct{} + +func (backend) Call(context.Context, string, service.Request) (any, error) { return nil, nil } +func TestReviewHTTPPairedQueueRoundTrip(t *testing.T) { + var notices atomic.Int32 + callback := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var p map[string]any + _ = json.NewDecoder(r.Body).Decode(&p) + if challenge, ok := p["challenge"].(string); ok { + _ = json.NewEncoder(w).Encode(map[string]string{"challenge": challenge}) + return + } + notices.Add(1) + w.WriteHeader(200) + })) + defer callback.Close() + tr := callback.Client().Transport.(*http.Transport).Clone() + tr.Proxy = nil + tr.TLSClientConfig.ServerName = "example.com" + tr.DialContext = func(ctx context.Context, network, address string) (net.Conn, error) { + return (&net.Dialer{}).DialContext(ctx, network, callback.Listener.Addr().String()) + } + delivery := dotevents.NewDelivery(&http.Client{Transport: tr, Timeout: 5 * time.Second, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }}) + v := dotbridge.NewFakeVault() + if e := dotbridge.EnsureStateKey(v); e != nil { + t.Fatal(e) + } + var paused atomic.Bool + s, e := dotbridge.Open(context.Background(), dotbridge.Config{Root: filepath.Join(t.TempDir(), "bridge"), Listen: "127.0.0.1:9317", Vault: v, MCPQueue: "device.distinct", EventsDelivery: delivery, Paused: paused.Load}) + if e != nil { + t.Fatal(e) + } + defer s.Close() + if e = s.Enable(); e != nil { + t.Fatal(e) + } + pair, _, e := s.Pair("") + if e != nil { + t.Fatal(e) + } + h, e := mcpserver.HTTPWithBridge(backend{}, s) + if e != nil { + t.Fatal(e) + } + const hop = "fixture-mcp-credential-32-characters-long" + mcpServer := httptest.NewServer(localapi.Guard(hop, h)) + defer mcpServer.Close() + rpc := func(method string, params map[string]any) json.RawMessage { + t.Helper() + params["_meta"] = map[string]any{"io.modelcontextprotocol/protocolVersion": "2026-07-28", "io.modelcontextprotocol/clientCapabilities": map[string]any{}, "io.modelcontextprotocol/clientInfo": map[string]any{"name": "fixture", "version": "1"}} + b, _ := json.Marshal(map[string]any{"jsonrpc": "2.0", "id": 1, "method": method, "params": params}) + req, _ := http.NewRequest("POST", mcpServer.URL, bytes.NewReader(b)) + req.Header.Set("X-DotControl-Token", hop) + req.Header.Set("Mcp-Method", method) + req.Header.Set("MCP-Protocol-Version", "2026-07-28") + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Accept", "application/json, text/event-stream") + if method == "tools/call" { + req.Header.Set("Mcp-Name", params["name"].(string)) + } + res, e := mcpServer.Client().Do(req) + if e != nil { + t.Fatal(e) + } + defer res.Body.Close() + raw, _ := io.ReadAll(res.Body) + var env struct { + Result json.RawMessage + Error json.RawMessage + } + if json.Unmarshal(raw, &env) != nil || res.StatusCode != 200 || len(env.Error) > 0 { + t.Fatalf("%s: %d %s", method, res.StatusCode, raw) + } + return env.Result + } + tool := func(name string, args any) json.RawMessage { + t.Helper() + raw := rpc("tools/call", map[string]any{"name": name, "arguments": args}) + var result struct { + IsError bool + StructuredContent json.RawMessage + } + if json.Unmarshal(raw, &result) != nil || result.IsError { + t.Fatalf("%s: %s", name, raw) + } + return result.StructuredContent + } + rpc("events/subscribe", map[string]any{"name": dotevents.EventName, "arguments": map[string]any{"queue_id": pair.QueueID}, "delivery": map[string]any{"mode": "webhook", "url": "https://callback.example/hook", "secret": "whsec_" + base64.StdEncoding.EncodeToString([]byte("01234567890123456789012345678901"))}}) + probe, e := s.StartProbe(pair.ClientID) + if e != nil { + t.Fatal(e) + } + deadline := time.Now().Add(5 * time.Second) + for notices.Load() == 0 && time.Now().Before(deadline) { + time.Sleep(10 * time.Millisecond) + } + if notices.Load() == 0 { + t.Fatal("no availability callback") + } + var claim dotqueue.Claim + if e = json.Unmarshal(tool("dot_task_claim", map[string]any{"queue_id": pair.QueueID, "job_id": probe.JobID, "worker_id": "attempt.1"}), &claim); e != nil { + t.Fatal(e) + } + answer := strings.TrimPrefix(claim.Submission.Messages[0].Content, "Reply with exactly this nonce: ") + tool("dot_task_status", map[string]any{"job_id": probe.JobID, "token": claim.Token}) + tool("dot_task_complete", map[string]any{"job_id": probe.JobID, "token": claim.Token, "result": dotqueue.Result{Text: answer}}) + api := httptest.NewServer(s.Handler()) + defer api.Close() + request := func(method, path string, body any) (int, []byte) { + t.Helper() + b, _ := json.Marshal(body) + req, _ := http.NewRequest(method, api.URL+path, bytes.NewReader(b)) + req.Header.Set("Authorization", "Bearer "+pair.Token) + res, e := api.Client().Do(req) + if e != nil { + t.Fatal(e) + } + defer res.Body.Close() + raw, _ := io.ReadAll(res.Body) + return res.StatusCode, raw + } + code, b := request("GET", "/dot-bridge/v1/status", nil) + var status dotbridge.StatusDTO + _ = json.Unmarshal(b, &status) + if code != 200 || !status.Ready || !status.Capabilities.Text || status.Capabilities.Tasks { + t.Fatalf("status: %d %s", code, b) + } + code, b = request("POST", "/dot-bridge/v1/jobs", map[string]any{"request_id": "normal.1", "session_id": "chat.1", "mode": "answer", "messages": []any{map[string]any{"role": "user", "content": "hello"}}}) + var job dotbridge.JobDTO + _ = json.Unmarshal(b, &job) + if code != 202 { + t.Fatalf("submit: %d %s", code, b) + } + _ = json.Unmarshal(tool("dot_task_claim", map[string]any{"queue_id": pair.QueueID, "job_id": job.JobID, "worker_id": "attempt.2"}), &claim) + tool("dot_task_complete", map[string]any{"job_id": job.JobID, "token": claim.Token, "result": dotqueue.Result{Text: "fixture answer"}}) + code, b = request("GET", "/dot-bridge/v1/jobs/"+job.JobID, nil) + if code != 200 || !bytes.Contains(b, []byte("fixture answer")) { + t.Fatalf("get: %d %s", code, b) + } + paused.Store(true) + _, b = request("GET", "/dot-bridge/v1/status", nil) + _ = json.Unmarshal(b, &status) + if status.Ready { + t.Fatal("ready while paused") + } +} diff --git a/internal/dotbridge/pairing.go b/internal/dotbridge/pairing.go index a92358a..2bf4079 100644 --- a/internal/dotbridge/pairing.go +++ b/internal/dotbridge/pairing.go @@ -25,13 +25,5 @@ func writePairingFile(root string, file PairingFile) (string, error) { } func EnsureStateKey(v vault.Store) error { - _, err := v.Get(StateKeyName) - if err == nil { - return nil - } - encoded, err := randomStateKey() - if err != nil { - return err - } - return v.Set(StateKeyName, encoded) + return vault.Ensure(v, StateKeyName, func() string { key, _ := randomStateKey(); return key }) } diff --git a/internal/dotbridge/probe.go b/internal/dotbridge/probe.go index 94f5403..0cb4404 100644 --- a/internal/dotbridge/probe.go +++ b/internal/dotbridge/probe.go @@ -32,7 +32,7 @@ func (s *Service) StartProbe(clientID string) (ProbeResult, error) { return ProbeResult{}, err } answer := hex.EncodeToString(nonce[:]) - requestID := "probe." + time.Now().UTC().Format("20060102150405") + requestID := "probe." + hashToken(answer) sub := dotqueue.Submission{ RequestID: requestID, Mode: dotqueue.ModeAnswer, @@ -46,15 +46,23 @@ func (s *Service) StartProbe(clientID string) (ProbeResult, error) { if err != nil { return ProbeResult{}, err } + if e := s.clientGate(clientID); e != nil { + return ProbeResult{}, e + } s.mu.Lock() + defer s.mu.Unlock() + policy, ok = s.state.Clients[clientID] + if !ok || policy.Revoked { + return ProbeResult{}, errors.New("access_revoked") + } policy.ProbeJobID = job.ID - policy.ProbeNonce = answer - policy.ProbeAnswer = answer + policy.ProbeAnswer = hashToken(answer) policy.ProbeVerified = time.Time{} policy.ProbeExpires = time.Time{} s.state.Clients[clientID] = policy - _ = s.saveState() - s.mu.Unlock() + if e := s.saveState(); e != nil { + return ProbeResult{}, e + } return ProbeResult{ JobID: job.ID, RequestID: requestID, diff --git a/internal/dotbridge/review_test.go b/internal/dotbridge/review_test.go new file mode 100644 index 0000000..eb70011 --- /dev/null +++ b/internal/dotbridge/review_test.go @@ -0,0 +1,78 @@ +package dotbridge + +import ( + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + + "github.com/dotcontrol/dotcontrol/internal/dotqueue" +) + +type interceptQueue struct { + QueueAPI + after func() +} + +func (q interceptQueue) Submit(id string, sub dotqueue.Submission) (dotqueue.Job, error) { + j, e := q.QueueAPI.Submit(id, sub) + q.after() + return j, e +} +func (q interceptQueue) Get(id, job string) (dotqueue.Job, error) { + j, e := q.QueueAPI.Get(id, job) + q.after() + return j, e +} + +func TestReviewCancelRoute(t *testing.T) { + s, token, id := openTestBridge(t) + p, e := s.StartProbe(id) + if e != nil { + t.Fatal(e) + } + r := httptest.NewRequest(http.MethodPost, "/dot-bridge/v1/jobs/"+p.JobID+"/cancel", nil) + r.Host = "127.0.0.1:9317" + r.Header.Set("Authorization", "Bearer "+token) + w := httptest.NewRecorder() + s.Handler().ServeHTTP(w, r) + if w.Code != 200 { + t.Fatalf("cancel: %d %s", w.Code, w.Body.String()) + } +} +func TestReviewProbeCannotUndoRevoke(t *testing.T) { + s, _, id := openTestBridge(t) + s.queue = interceptQueue{QueueAPI: s.queue, after: func() { + if e := s.RevokeClient(id); e != nil { + t.Fatal(e) + } + }} + if _, e := s.StartProbe(id); e == nil { + t.Fatal("probe reported success after revocation") + } + s.mu.Lock() + revoked := s.state.Clients[id].Revoked + s.mu.Unlock() + if !revoked { + t.Fatal("probe resurrected revoked policy") + } +} +func TestReviewReadRechecksPause(t *testing.T) { + s, _, id := openTestBridge(t) + p, e := s.StartProbe(id) + if e != nil { + t.Fatal(e) + } + var paused atomic.Bool + s.paused = paused.Load + s.queue = interceptQueue{QueueAPI: s.queue, after: func() { paused.Store(true) }} + if _, e := s.getJob(ClientPolicy{ClientID: id}, p.JobID); e == nil { + t.Fatal("read succeeded after pause during I/O") + } +} +func TestReviewStrictSubmit(t *testing.T) { + if _, e := decodeSubmit(strings.NewReader(`{"request_id":"r","mode":"answer","session_id":"s","messages":[{"role":"user","content":"hi"}]}`)); e != nil { + t.Fatal(e) + } +} diff --git a/internal/dotbridge/service.go b/internal/dotbridge/service.go index 46cf25e..829167d 100644 --- a/internal/dotbridge/service.go +++ b/internal/dotbridge/service.go @@ -16,21 +16,22 @@ import ( // Service owns bridge lifecycle, local API auth, and delivery worker wiring. type Service struct { - mu sync.Mutex - ctx context.Context - cancel context.CancelFunc - root string - listen string - vault vault.Store - queue QueueAPI - events EventsAPI - mcpQueue string - mcpEpoch uint64 - state PersistedState - recovered bool - worker *worker - paused func() bool - revoked func() bool + mu sync.Mutex + ctx context.Context + cancel context.CancelFunc + root string + listen string + vault vault.Store + queue QueueAPI + events EventsAPI + mcpQueue string + subscriptions map[string]subscriptionProof + claims map[string]claimBinding + state PersistedState + recovered bool + worker *worker + paused func() bool + revoked func() bool } type Config struct { @@ -38,7 +39,6 @@ type Config struct { Listen string Vault vault.Store MCPQueue string - MCPEpoch uint64 Paused func() bool Revoked func() bool EventsDelivery *dotevents.Delivery @@ -72,18 +72,19 @@ func Open(ctx context.Context, cfg Config) (*Service, error) { } ctx, cancel := context.WithCancel(ctx) s := &Service{ - ctx: ctx, - cancel: cancel, - root: cfg.Root, - listen: cfg.Listen, - vault: cfg.Vault, - queue: wrapQueue(queueStore), - events: wrapEvents(eventsMgr), - mcpQueue: cfg.MCPQueue, - mcpEpoch: cfg.MCPEpoch, - paused: cfg.Paused, - revoked: cfg.Revoked, - state: PersistedState{Version: APIVersion, Clients: map[string]ClientPolicy{}}, + ctx: ctx, + cancel: cancel, + root: cfg.Root, + listen: cfg.Listen, + vault: cfg.Vault, + queue: wrapQueue(queueStore), + events: wrapEvents(eventsMgr), + mcpQueue: cfg.MCPQueue, + subscriptions: map[string]subscriptionProof{}, + claims: map[string]claimBinding{}, + paused: cfg.Paused, + revoked: cfg.Revoked, + state: PersistedState{Version: APIVersion, Clients: map[string]ClientPolicy{}}, } if err := s.loadState(); err != nil { cancel() @@ -138,7 +139,10 @@ func (s *Service) gate() error { if s.paused != nil && s.paused() { return errors.New("bridge_paused") } - if !s.state.Enabled { + s.mu.Lock() + enabled := s.state.Enabled + s.mu.Unlock() + if !enabled { return errors.New("bridge_disabled") } return nil @@ -195,51 +199,47 @@ func (s *Service) AdminStatus() (map[string]any, error) { func (s *Service) StatusForClient(clientID string) (StatusDTO, error) { s.mu.Lock() - defer s.mu.Unlock() policy, ok := s.state.Clients[clientID] + enabled := s.state.Enabled + s.mu.Unlock() if !ok || policy.Revoked { - return StatusDTO{Version: APIVersion, Enabled: s.state.Enabled, Paired: false, Ready: false, Code: "not_paired", Capabilities: CapPolicy{}}, nil + return StatusDTO{Version: APIVersion, Enabled: enabled, Code: "not_paired"}, nil } return s.statusDTO(policy), nil } - func (s *Service) statusDTO(policy ClientPolicy) StatusDTO { - now := time.Now().UTC() - eventsStatus, _ := s.events.Status(policy.QueueID, policy.QueueID) - sub := SubscriptionDTO{State: "inactive"} - if eventsStatus.Subscribed && eventsStatus.CallbackVerified { - sub.State = "active" - } - ready := false - code := "probe_required" - var verifiedAt, expiresAt *time.Time - if s.state.Enabled && policy.ProbeVerified.After(time.Time{}) && policy.ProbeExpires.After(now) { - ready = eventsStatus.Subscribed && eventsStatus.CallbackVerified - if ready { - code = "ready" - verifiedAt = &policy.ProbeVerified - expiresAt = &policy.ProbeExpires - if sub.State == "active" { - sub.ExpiresAt = expiresAt - } + s.mu.Lock() + enabled := s.state.Enabled + current := s.state.Clients[policy.ClientID] + s.mu.Unlock() + sub, active := s.activeSubscription(policy.QueueID) + eventStatus, e := s.events.Status(policy.QueueID, policy.QueueID) + ready := s.clientGate(policy.ClientID) == nil && active && e == nil && eventStatus.Subscribed && eventStatus.CallbackVerified && current.ProbeVerified.After(time.Time{}) && time.Now().Before(current.ProbeExpires) + out := StatusDTO{Version: APIVersion, Enabled: enabled, Paired: !current.Revoked, Code: "probe_required", Subscription: SubscriptionDTO{State: "inactive"}} + if active { + out.Subscription = SubscriptionDTO{State: "active", ExpiresAt: &sub.Expires} + } + if ready { + expiry := current.ProbeExpires + if sub.Expires.Before(expiry) { + expiry = sub.Expires } + out.Ready = true + out.Code = "ready" + out.VerifiedAt = ¤t.ProbeVerified + out.ExpiresAt = &expiry + out.Capabilities = CapPolicy{Text: true} } - return StatusDTO{ - Version: APIVersion, - Enabled: s.state.Enabled, - Paired: true, - Ready: ready, - Code: code, - VerifiedAt: verifiedAt, - ExpiresAt: expiresAt, - Subscription: sub, - Capabilities: policy.Capabilities, - } + return out } func (s *Service) Pair(clientID string) (PairingFile, string, error) { if clientID == "" { - clientID = "client." + time.Now().UTC().Format("20060102150405") + id, e := randomToken() + if e != nil { + return PairingFile{}, "", e + } + clientID = "client." + id } token, err := randomToken() if err != nil { @@ -247,6 +247,9 @@ func (s *Service) Pair(clientID string) (PairingFile, string, error) { } s.mu.Lock() defer s.mu.Unlock() + if _, exists := s.state.Clients[clientID]; exists { + return PairingFile{}, "", errors.New("client_exists") + } policy := ClientPolicy{ ClientID: clientID, QueueID: clientID, @@ -272,6 +275,7 @@ func (s *Service) RevokeClient(clientID string) error { } policy.Revoked = true s.state.Clients[clientID] = policy + delete(s.subscriptions, policy.QueueID) return s.saveState() } @@ -297,21 +301,52 @@ func (s *Service) MCPPrincipal() (string, string, error) { } func (s *Service) EventsList(principal string) []dotevents.Definition { - return s.events.List(principal) + if s.gate() != nil { + return nil + } + return []dotevents.Definition{{Name: dotevents.EventName, Description: "Availability notices for an authorized paired queue."}} } - func (s *Service) EventsSubscribe(ctx context.Context, principal string, params dotevents.Params) (dotevents.Subscription, error) { - if err := s.gate(); err != nil { - return dotevents.Subscription{}, err + queue := params.Arguments.QueueID + if e := s.authorizeQueue(queue); e != nil { + return dotevents.Subscription{}, e } - return s.events.Subscribe(ctx, principal, params) + sub, e := s.events.Subscribe(ctx, queue, params) + if e != nil { + return sub, e + } + if e = s.authorizeQueue(queue); e != nil { + return dotevents.Subscription{}, e + } + s.mu.Lock() + defer s.mu.Unlock() + p := s.state.Clients[queue] + if p.Revoked { + return dotevents.Subscription{}, errors.New("access_revoked") + } + old := s.subscriptions[queue] + s.subscriptions[queue] = subscriptionProof{Expires: sub.ExpiresAt, Generation: old.Generation + 1} + p.ProbeVerified = time.Time{} + p.ProbeExpires = time.Time{} + s.state.Clients[queue] = p + return sub, nil } - func (s *Service) EventsUnsubscribe(ctx context.Context, principal string, params dotevents.Params) error { - if err := s.gate(); err != nil { - return err + queue := params.Arguments.QueueID + if e := s.authorizeQueue(queue); e != nil { + return e } - return s.events.Unsubscribe(ctx, principal, params) + e := s.events.Unsubscribe(ctx, queue, params) + s.mu.Lock() + old := s.subscriptions[queue] + old.Expires = time.Time{} + old.Generation++ + s.subscriptions[queue] = old + s.mu.Unlock() + if post := s.authorizeQueue(queue); post != nil { + return post + } + return e } func (s *Service) ClaimTask(queueID, jobID, workerID string) (dotqueue.Claim, error) { @@ -366,9 +401,12 @@ func (s *Service) submit(client ClientPolicy, req SubmitRequest) (JobDTO, int, e if err := s.gate(); err != nil { return JobDTO{}, 0, err } + if e := s.clientGate(client.ClientID); e != nil { + return JobDTO{}, 0, e + } status := s.statusDTO(client) - if !status.Ready && req.Mode != dotqueue.ModeAnswer { - return JobDTO{}, 0, errors.New("probe_required") + if req.Mode == dotqueue.ModeTask { + return JobDTO{}, 0, errors.New("task_verification_required") } if !status.Ready { return JobDTO{}, 0, errors.New("probe_required") @@ -386,6 +424,9 @@ func (s *Service) submit(client ClientPolicy, req SubmitRequest) (JobDTO, int, e if err != nil { return JobDTO{}, 0, err } + if e := s.clientGate(client.ClientID); e != nil { + return JobDTO{}, 0, e + } dto := toJobDTO(job) dto.Accepted = true return dto, 202, nil @@ -395,10 +436,16 @@ func (s *Service) getJob(client ClientPolicy, jobID string) (JobDTO, error) { if err := s.gate(); err != nil { return JobDTO{}, err } + if e := s.clientGate(client.ClientID); e != nil { + return JobDTO{}, e + } job, err := s.queue.Get(client.ClientID, jobID) if err != nil { return JobDTO{}, err } + if e := s.clientGate(client.ClientID); e != nil { + return JobDTO{}, e + } return toJobDTO(job), nil } @@ -406,10 +453,16 @@ func (s *Service) byRequest(client ClientPolicy, requestID string) (JobDTO, erro if err := s.gate(); err != nil { return JobDTO{}, err } + if e := s.clientGate(client.ClientID); e != nil { + return JobDTO{}, e + } job, err := s.queue.ByRequest(client.ClientID, requestID) if err != nil { return JobDTO{}, err } + if e := s.clientGate(client.ClientID); e != nil { + return JobDTO{}, e + } return toJobDTO(job), nil } @@ -417,10 +470,16 @@ func (s *Service) cancelJob(client ClientPolicy, jobID string) (JobDTO, error) { if err := s.gate(); err != nil { return JobDTO{}, err } + if e := s.clientGate(client.ClientID); e != nil { + return JobDTO{}, e + } job, err := s.queue.Cancel(client.ClientID, jobID) if err != nil { return JobDTO{}, err } + if e := s.clientGate(client.ClientID); e != nil { + return JobDTO{}, e + } return toJobDTO(job), nil } diff --git a/internal/dotbridge/types.go b/internal/dotbridge/types.go index d4ecf5c..e91c086 100644 --- a/internal/dotbridge/types.go +++ b/internal/dotbridge/types.go @@ -3,10 +3,9 @@ package dotbridge import "time" const ( - APIVersion = 1 - StateKeyName = "dot-bridge-state-key" - probeValidity = 10 * time.Minute - statusCacheMaxAge = 10 * time.Second + APIVersion = 1 + StateKeyName = "dot-bridge-state-key" + probeValidity = 10 * time.Minute ) type ClientPolicy struct { @@ -17,8 +16,7 @@ type ClientPolicy struct { Revoked bool `json:"revoked"` Capabilities CapPolicy `json:"capabilities"` ProbeJobID string `json:"probe_job_id,omitempty"` - ProbeNonce string `json:"probe_nonce,omitempty"` - ProbeAnswer string `json:"probe_answer,omitempty"` + ProbeAnswer string `json:"probe_answer_hash,omitempty"` ProbeVerified time.Time `json:"probe_verified_at,omitempty"` ProbeExpires time.Time `json:"probe_expires_at,omitempty"` } diff --git a/internal/dotbridge/worker.go b/internal/dotbridge/worker.go index 8140dd7..c3f6187 100644 --- a/internal/dotbridge/worker.go +++ b/internal/dotbridge/worker.go @@ -2,6 +2,7 @@ package dotbridge import ( "context" + "errors" "sync" "time" @@ -55,7 +56,7 @@ func (w *worker) tick() { return } for _, queueID := range queues { - if err := w.svc.gate(); err != nil { + if err := w.svc.authorizeQueue(queueID); err != nil { return } notices, err := w.svc.queue.Pending(queueID) @@ -63,7 +64,7 @@ func (w *worker) tick() { continue } for _, notice := range notices { - if err := w.svc.gate(); err != nil { + if err := w.svc.authorizeQueue(queueID); err != nil { return } ctx, cancel := context.WithTimeout(w.svc.ctx, 5*time.Second) @@ -75,6 +76,9 @@ func (w *worker) tick() { OccurredAt: notice.OccurredAt, }) cancel() + if w.svc.authorizeQueue(queueID) != nil { + return + } if err != nil { continue } @@ -83,57 +87,87 @@ func (w *worker) tick() { } func (s *Service) claimTask(queueID, jobID, workerID string) (dotqueue.Claim, error) { - if err := s.gate(); err != nil { - return dotqueue.Claim{}, err + if e := s.authorizeQueue(queueID); e != nil { + return dotqueue.Claim{}, e + } + if workerID == "" { + return dotqueue.Claim{}, errors.New("worker_required") + } + sub, active := s.activeSubscription(queueID) + if !active { + return dotqueue.Claim{}, errors.New("subscription_required") } - authorized, _, err := s.MCPPrincipal() - if err != nil { - return dotqueue.Claim{}, err + claim, e := s.queue.Claim(queueID, jobID, consumerWorker(s.mcpQueue, workerID)) + if e != nil { + return claim, e + } + if e = s.authorizeQueue(queueID); e != nil { + return dotqueue.Claim{}, e + } + s.mu.Lock() + defer s.mu.Unlock() + if s.state.Clients[queueID].Revoked { + return dotqueue.Claim{}, errors.New("access_revoked") } - if queueID != authorized { - return dotqueue.Claim{}, &dotqueue.Error{Code: "unknown"} + if _, exists := s.claims[jobID]; !exists { + s.claims[jobID] = claimBinding{QueueID: queueID, Token: claim.Token, Generation: sub.Generation} } - return s.queue.Claim(queueID, jobID, workerID) + return claim, nil } - func (s *Service) progressTask(jobID, token, state, summary string) (dotqueue.Job, error) { - if err := s.gate(); err != nil { - return dotqueue.Job{}, err + b, e := s.binding(jobID, token) + if e != nil { + return dotqueue.Job{}, e + } + j, e := s.queue.Progress(jobID, token, state, summary) + if post := s.authorizeQueue(b.QueueID); post != nil { + return dotqueue.Job{}, post } - return s.queue.Progress(jobID, token, state, summary) + return j, e } - func (s *Service) completeTask(jobID, token string, result dotqueue.Result) (dotqueue.Job, error) { - if err := s.gate(); err != nil { - return dotqueue.Job{}, err + b, e := s.binding(jobID, token) + if e != nil { + return dotqueue.Job{}, e + } + job, e := s.queue.Complete(jobID, token, result) + if e != nil { + return dotqueue.Job{}, e } - job, err := s.queue.Complete(jobID, token, result) - if err != nil { - return job, err + if e = s.authorizeQueue(b.QueueID); e != nil { + return dotqueue.Job{}, e } s.mu.Lock() - for id, policy := range s.state.Clients { - if policy.ProbeJobID == jobID && policy.ProbeAnswer != "" && result.Text == policy.ProbeAnswer { - now := time.Now().UTC() - policy.ProbeVerified = now - policy.ProbeExpires = now.Add(probeValidity) - policy.Capabilities.Text = true - s.state.Clients[id] = policy - _ = s.saveState() - break + defer s.mu.Unlock() + sub := s.subscriptions[b.QueueID] + active := time.Now().Before(sub.Expires) + policy := s.state.Clients[b.QueueID] + if policy.Revoked { + return dotqueue.Job{}, errors.New("access_revoked") + } + if active && sub.Generation == b.Generation && policy.ProbeJobID == jobID && policy.ProbeAnswer == hashToken(result.Text) && job.State == dotqueue.StateCompleted && job.Mode == dotqueue.ModeAnswer { + now := time.Now().UTC() + policy.ProbeVerified = now + policy.ProbeExpires = now.Add(probeValidity) + if sub.Expires.Before(policy.ProbeExpires) { + policy.ProbeExpires = sub.Expires + } + policy.Capabilities = CapPolicy{Text: true} + s.state.Clients[b.QueueID] = policy + if e := s.saveState(); e != nil { + return dotqueue.Job{}, e } } - s.mu.Unlock() return job, nil } - func (s *Service) taskStatus(principal, jobID, token string) (dotqueue.Job, error) { - if err := s.gate(); err != nil { - return dotqueue.Job{}, err + b, e := s.binding(jobID, token) + if e != nil { + return dotqueue.Job{}, e } - job, err := s.queue.Get(principal, jobID) - if err != nil { - return dotqueue.Job{}, err + job, e := s.queue.Get(b.QueueID, jobID) + if post := s.authorizeQueue(b.QueueID); post != nil { + return dotqueue.Job{}, post } - return s.queue.Progress(jobID, token, job.State, job.Summary) + return job, e } diff --git a/internal/mcpserver/bridge.go b/internal/mcpserver/bridge.go index a61b052..81665ca 100644 --- a/internal/mcpserver/bridge.go +++ b/internal/mcpserver/bridge.go @@ -2,6 +2,7 @@ package mcpserver import ( "context" + "encoding/json" "fmt" "net/http" @@ -26,9 +27,9 @@ type TaskProgressArgs struct { } type TaskCompleteArgs struct { - JobID string `json:"job_id"` - Token string `json:"token"` - Result dotqueue.Result `json:"result"` + JobID string `json:"job_id"` + Token string `json:"token"` + Result dotqueue.Result `json:"result"` } type TaskStatusArgs struct { @@ -47,7 +48,11 @@ type EventsListResult struct { type EventsSubscribeParams struct { mcp.ParamsBase - Params dotevents.Params `json:"params"` + Name string `json:"name"` + Arguments dotevents.Arguments `json:"arguments"` + Delivery dotevents.Callback `json:"delivery"` + Cursor json.RawMessage `json:"cursor,omitempty"` + TTL json.RawMessage `json:"ttlMs,omitempty"` } type EventsSubscribeResult struct { @@ -57,7 +62,11 @@ type EventsSubscribeResult struct { type EventsUnsubscribeParams struct { mcp.ParamsBase - Params dotevents.Params `json:"params"` + Name string `json:"name"` + Arguments dotevents.Arguments `json:"arguments"` + Delivery dotevents.Callback `json:"delivery"` + Cursor json.RawMessage `json:"cursor,omitempty"` + TTL json.RawMessage `json:"ttlMs,omitempty"` } type EventsUnsubscribeResult struct { @@ -115,10 +124,8 @@ func NewWithBridge(backend service.Backend, bridge *dotbridge.Service) (*mcp.Ser if err != nil { return nil, err } - if params.Params.Arguments.QueueID != queueID { - return nil, fmt.Errorf("queue_mismatch") - } - sub, err := bridge.EventsSubscribe(ctx, principal, params.Params) + _ = queueID + sub, err := bridge.EventsSubscribe(ctx, principal, dotevents.Params{Name: params.Name, Arguments: params.Arguments, Delivery: params.Delivery, Cursor: params.Cursor, TTL: params.TTL}) if err != nil { return nil, err } @@ -131,10 +138,8 @@ func NewWithBridge(backend service.Backend, bridge *dotbridge.Service) (*mcp.Ser if err != nil { return nil, err } - if params.Params.Arguments.QueueID != queueID { - return nil, fmt.Errorf("queue_mismatch") - } - if err := bridge.EventsUnsubscribe(ctx, principal, params.Params); err != nil { + _ = queueID + if err := bridge.EventsUnsubscribe(ctx, principal, dotevents.Params{Name: params.Name, Arguments: params.Arguments, Delivery: params.Delivery, Cursor: params.Cursor, TTL: params.TTL}); err != nil { return nil, err } return &EventsUnsubscribeResult{}, nil diff --git a/internal/mcpserver/bridge_http_test.go b/internal/mcpserver/bridge_http_test.go new file mode 100644 index 0000000..ccfca1a --- /dev/null +++ b/internal/mcpserver/bridge_http_test.go @@ -0,0 +1,55 @@ +package mcpserver + +import ( + "bytes" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "testing" +) + +func TestReviewBridgeHTTPToolsAndDiscovery(t *testing.T) { + s := openBridgeForMCPTest(t) + h, e := HTTPWithBridge(bridgeNoopBackend{}, s) + if e != nil { + t.Fatal(e) + } + server := httptest.NewServer(h) + defer server.Close() + for _, method := range []string{"tools/list", "server/discover"} { + req, _ := http.NewRequest("POST", server.URL, bytes.NewBufferString(`{"jsonrpc":"2.0","id":1,"method":"`+method+`","params":{"_meta":{"io.modelcontextprotocol/protocolVersion":"2026-07-28","io.modelcontextprotocol/clientCapabilities":{},"io.modelcontextprotocol/clientInfo":{"name":"fixture","version":"1"}}}}`)) + req.Header.Set("Mcp-Method", method) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Accept", "application/json, text/event-stream") + req.Header.Set("MCP-Protocol-Version", "2026-07-28") + res, e := server.Client().Do(req) + if e != nil { + t.Fatalf("%s: %v", method, e) + } + b, _ := io.ReadAll(res.Body) + res.Body.Close() + if res.StatusCode != 200 { + t.Fatalf("%s: %d %s", method, res.StatusCode, b) + } + var env struct { + Result struct { + Tools []any `json:"tools"` + Capabilities map[string]json.RawMessage `json:"capabilities"` + } + Error any + } + if e := json.Unmarshal(b, &env); e != nil { + t.Fatal(e) + } + if env.Error != nil { + t.Fatalf("%s: %s", method, b) + } + if method == "tools/list" && len(env.Result.Tools) != 17 { + t.Fatalf("tools: %s", b) + } + if method == "server/discover" && string(env.Result.Capabilities["events"]) != "{}" { + t.Fatalf("discovery: %s", b) + } + } +} diff --git a/internal/mcpserver/discover.go b/internal/mcpserver/discover.go index db493c2..d5f605a 100644 --- a/internal/mcpserver/discover.go +++ b/internal/mcpserver/discover.go @@ -7,18 +7,18 @@ import ( "net/http" ) -// DiscoverEventsWrapper augments successful 2026-07-28 server/discover responses -// with events capability without replacing SDK dispatch. +// DiscoverEventsWrapper only augments a successful modern SDK discovery result. +// Other traffic remains on the original SDK ResponseWriter (including streaming). func DiscoverEventsWrapper(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.Method != http.MethodPost { + if r.Method != "POST" || r.Header.Get("MCP-Protocol-Version") != "2026-07-28" || r.Header.Get("Mcp-Method") != "server/discover" { next.ServeHTTP(w, r) return } rec := &responseRecorder{header: make(http.Header)} next.ServeHTTP(rec, r) body := rec.body.Bytes() - if rec.status == http.StatusOK && bytes.Contains(body, []byte(`"method":"server/discover"`)) { + if rec.status == 200 { body = patchDiscoverEvents(body) } for k, vals := range rec.header { @@ -26,7 +26,12 @@ func DiscoverEventsWrapper(next http.Handler) http.Handler { w.Header().Add(k, v) } } - w.WriteHeader(rec.status) + w.Header().Del("Content-Length") + status := rec.status + if status == 0 { + status = 200 + } + w.WriteHeader(status) _, _ = io.Copy(w, bytes.NewReader(body)) }) } @@ -38,39 +43,48 @@ type responseRecorder struct { } func (r *responseRecorder) Header() http.Header { return r.header } -func (r *responseRecorder) Write(b []byte) (int, error) { return r.body.Write(b) } -func (r *responseRecorder) WriteHeader(statusCode int) { r.status = statusCode } - +func (r *responseRecorder) Write(b []byte) (int, error) { + if r.status == 0 { + r.WriteHeader(200) + } + return r.body.Write(b) +} +func (r *responseRecorder) WriteHeader(code int) { + if r.status == 0 { + r.status = code + } +} func patchDiscoverEvents(body []byte) []byte { var env map[string]json.RawMessage - if json.Unmarshal(body, &env) != nil { + if json.Unmarshal(body, &env) != nil || env["error"] != nil || env["id"] == nil { return body } - result, ok := env["result"] - if !ok { + var typed struct { + SupportedVersions []string `json:"supportedVersions"` + Capabilities *json.RawMessage `json:"capabilities"` + } + if json.Unmarshal(env["result"], &typed) != nil || typed.Capabilities == nil { return body } - var obj map[string]any - if json.Unmarshal(result, &obj) != nil { + modern := false + for _, v := range typed.SupportedVersions { + if v == "2026-07-28" { + modern = true + } + } + if !modern { return body } - caps, ok := obj["capabilities"].(map[string]any) - if !ok { + var result, caps map[string]json.RawMessage + if json.Unmarshal(env["result"], &result) != nil || json.Unmarshal(*typed.Capabilities, &caps) != nil || caps == nil { return body } - if _, exists := caps["events"]; !exists { - caps["events"] = map[string]any{} - obj["capabilities"] = caps - patched, err := json.Marshal(obj) - if err != nil { - return body - } - env["result"] = patched - out, err := json.Marshal(env) - if err != nil { - return body - } - return out + caps["events"] = json.RawMessage(`{}`) + result["capabilities"], _ = json.Marshal(caps) + env["result"], _ = json.Marshal(result) + out, e := json.Marshal(env) + if e != nil { + return body } - return body + return out } diff --git a/internal/mcpserver/discover_test.go b/internal/mcpserver/discover_test.go new file mode 100644 index 0000000..8b712c3 --- /dev/null +++ b/internal/mcpserver/discover_test.go @@ -0,0 +1,23 @@ +package mcpserver + +import ( + "bytes" + "testing" +) + +func TestDiscoverPatchOnlySuccessfulModernTypedResult(t *testing.T) { + for _, raw := range []string{ + `{"jsonrpc":"2.0","id":1,"error":{"code":-1,"message":"failed"}}`, + `{"jsonrpc":"2.0","id":1,"result":{"supportedVersions":["2025-11-25"],"capabilities":{"tools":{}}}}`, + `{"jsonrpc":"2.0","id":1,"result":{"tools":[]}}`, + } { + if got := patchDiscoverEvents([]byte(raw)); !bytes.Equal(got, []byte(raw)) { + t.Fatalf("changed unsuccessful or unrelated result: %s", got) + } + } + raw := `{"jsonrpc":"2.0","id":9007199254740993,"result":{"supportedVersions":["2026-07-28"],"capabilities":{"tools":{}},"futureNumber":9007199254740993}}` + got := patchDiscoverEvents([]byte(raw)) + if !bytes.Contains(got, []byte(`"events":{}`)) || bytes.Count(got, []byte("9007199254740993")) != 2 { + t.Fatalf("patched result: %s", got) + } +} From 41f2748b3e21e2a405ffab484c92f29035cadfdb Mon Sep 17 00:00:00 2001 From: Pyuyi <204892921+VeraPyuyi@users.noreply.github.com> Date: Mon, 19 Oct 2026 13:59:43 +0800 Subject: [PATCH 11/47] feat(dotqueue): add scoped List for owner job history --- docs/DOT_TASK_QUEUE.md | 2 +- .../report/research/dot-queue-list-report.md | 41 ++++ internal/dotqueue/list.go | 39 ++++ internal/dotqueue/list_test.go | 177 ++++++++++++++++++ 4 files changed, 258 insertions(+), 1 deletion(-) create mode 100644 evidence/report/research/dot-queue-list-report.md create mode 100644 internal/dotqueue/list.go create mode 100644 internal/dotqueue/list_test.go diff --git a/docs/DOT_TASK_QUEUE.md b/docs/DOT_TASK_QUEUE.md index 3090d47..ac24680 100644 --- a/docs/DOT_TASK_QUEUE.md +++ b/docs/DOT_TASK_QUEUE.md @@ -2,7 +2,7 @@ `internal/dotqueue` is a trusted local Go backend. It persists work and stable event notices; it does not expose routing, implement authentication, fetch artifact URLs, or establish Dot readiness. -Call `Open(dir, key)` with a caller-managed 32-byte key and explicitly call `Initialize(enabled)` for a new store. Open copies the key and never persists it or automatically recovers work. Initialize on an existing valid store sets enabled; a missing, malformed, or unsupported initialized marker/state fails closed. `SetEnabled` changes the persisted gate. `ConfigStatus()` returns only `Enabled` and remains available while disabled. Get, ByRequest, Pending, Submit, Claim, Progress, Complete, Cancel and Recover all check enabled under the transaction lock. +Call `Open(dir, key)` with a caller-managed 32-byte key and explicitly call `Initialize(enabled)` for a new store. Open copies the key and never persists it or automatically recovers work. Initialize on an existing valid store sets enabled; a missing, malformed, or unsupported initialized marker/state fails closed. `SetEnabled` changes the persisted gate. `ConfigStatus()` returns only `Enabled` and remains available while disabled. Get, ByRequest, List, Pending, Submit, Claim, Progress, Complete, Cancel and Recover all check enabled under the transaction lock. List returns only the owning clientID jobs as public Job values in newest-first creation order (reverse of persisted order), omitting claim tokens and original submission payload, without mutating stored records. Submission IDs (including client, queue, worker and optional session IDs) use `^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$`. Answer mode requires messages; task mode requires nonblank Task. The supported message roles are system, user and assistant. At most 32 messages are accepted. Combined Task and message content is at most 262144 UTF-8 bytes. Invalid UTF-8 is rejected before hashing. Canonical JSON uses fixed field order and empty arrays for nil message/artifact lists; SHA-256 identifies the complete submission/result. diff --git a/evidence/report/research/dot-queue-list-report.md b/evidence/report/research/dot-queue-list-report.md new file mode 100644 index 0000000..3c65f5f --- /dev/null +++ b/evidence/report/research/dot-queue-list-report.md @@ -0,0 +1,41 @@ +# Dot queue List slice report + +## Scope + +Implemented `Store.List(clientID string) ([]Job, error)` in `internal/dotqueue/list.go` with behavioral tests in `internal/dotqueue/list_test.go` and documentation in `docs/DOT_TASK_QUEUE.md`. + +## Behavior + +- Validates `clientID` with `validateID` before locking. +- Uses `withLock(false, false)` and rejects when `!st.Enabled` (`dot_queue_disabled`). +- Iterates persisted `order` in reverse so results are newest-first by creation order. +- Returns only jobs whose `client_id` matches the caller; foreign clients never appear in another owner's list. +- Maps each record through `copyPublicJob`, which builds a public `Job` and deep-copies `Result` (including artifacts) so returned values do not alias internal `jobRecord` fields. +- Does not mutate `jobRecord` or persistence state during listing. +- Retention remains bounded by the existing 100-job store limit; List does not evict or create jobs. +- On error returns `(nil, err)`; on success returns a non-nil empty slice when the owner has zero jobs. + +## Tests (RED/GREEN) + +| Test | Intent | +|------|--------| +| `TestListRejectsInvalidClientID` | Empty client ID fails before read | +| `TestListDisabledFailsClosed` | Disabled gate matches Get/ByRequest | +| `TestListOwnerForeignAndZero` | Empty owner list, foreign isolation, newest-first order | +| `TestListIncludesTerminalStatesAndStableReopen` | Cancelled/completed included; order stable after reopen | +| `TestListOmitsSensitiveFieldsAndDoesNotAlias` | JSON omits prompt/token/payload fields; copies do not alias store | + +## Verification commands + +Working directory: `D:\Applications\Opencode\data\share\worktree\bd2540\dot-task-event-bridge` + +```powershell +& "C:\Program Files\Go\bin\go.exe" test ./internal/dotqueue -count=1 -run TestList -v +& "C:\Program Files\Go\bin\go.exe" test -race ./internal/dotqueue -count=1 -run TestList +``` + +Results: all `TestList*` cases passed on 2026-10-19. `-race` was attempted with `CGO_ENABLED=1` but the environment has no C compiler (`gcc` not found); rerun `-race` where CGO and a compiler are available. + +## Follow-up + +HTTP/controller `list` GET adapter wiring is intentionally out of scope for this slice. diff --git a/internal/dotqueue/list.go b/internal/dotqueue/list.go new file mode 100644 index 0000000..4ea523e --- /dev/null +++ b/internal/dotqueue/list.go @@ -0,0 +1,39 @@ +package dotqueue + +func (s *Store) List(clientID string) ([]Job, error) { + if err := validateID(clientID); err != nil { + return nil, err + } + out := make([]Job, 0) + err := s.withLock(false, false, func(st *persistenceState) error { + if !st.Enabled { + return errf("disabled") + } + for i := len(st.Order) - 1; i >= 0; i-- { + id := st.Order[i] + rec := st.Jobs[id] + if rec == nil || rec.ClientID != clientID { + continue + } + out = append(out, copyPublicJob(rec)) + } + return nil + }) + if err != nil { + return nil, err + } + return out, nil +} + +func copyPublicJob(rec *jobRecord) Job { + job := publicJob(rec) + if rec.Result != nil { + job.Result = &Result{ + Text: rec.Result.Text, + Artifacts: append([]Artifact(nil), rec.Result.Artifacts...), + ErrorCode: rec.Result.ErrorCode, + Cancelled: rec.Result.Cancelled, + } + } + return job +} diff --git a/internal/dotqueue/list_test.go b/internal/dotqueue/list_test.go new file mode 100644 index 0000000..80c49bf --- /dev/null +++ b/internal/dotqueue/list_test.go @@ -0,0 +1,177 @@ +package dotqueue + +import ( + "encoding/json" + "strings" + "testing" +) + +func TestListRejectsInvalidClientID(t *testing.T) { + store, _ := openTestStore(t) + jobs, err := store.List("") + if err == nil || jobs != nil { + t.Fatalf("expected invalid client rejection, got jobs=%#v err=%v", jobs, err) + } + if errorCode(err) != "unknown" { + t.Fatalf("expected unknown, got %v", err) + } +} + +func TestListDisabledFailsClosed(t *testing.T) { + store, _ := openTestStore(t) + if _, err := store.Submit("client.a", sampleSubmission("req.list.disabled")); err != nil { + t.Fatal(err) + } + if err := store.SetEnabled(false); err != nil { + t.Fatal(err) + } + jobs, err := store.List("client.a") + if err == nil || jobs != nil { + t.Fatalf("expected disabled list failure, got jobs=%#v err=%v", jobs, err) + } + if errorCode(err) != "disabled" { + t.Fatalf("expected disabled, got %v", err) + } +} + +func TestListOwnerForeignAndZero(t *testing.T) { + store, _ := openTestStore(t) + zero, err := store.List("client.empty") + if err != nil { + t.Fatal(err) + } + if zero == nil || len(zero) != 0 { + t.Fatalf("expected empty owner list, got %#v", zero) + } + + first, err := store.Submit("client.a", sampleSubmission("req.list.owner.1")) + if err != nil { + t.Fatal(err) + } + second, err := store.Submit("client.a", sampleSubmission("req.list.owner.2")) + if err != nil { + t.Fatal(err) + } + if _, err := store.Submit("client.b", sampleSubmission("req.list.foreign")); err != nil { + t.Fatal(err) + } + + listed, err := store.List("client.a") + if err != nil { + t.Fatal(err) + } + if len(listed) != 2 { + t.Fatalf("expected two owner jobs, got %#v", listed) + } + if listed[0].ID != second.ID || listed[1].ID != first.ID { + t.Fatalf("expected newest-first order, got %#v", listed) + } + for _, job := range listed { + if job.ClientID != "client.a" { + t.Fatalf("foreign job leaked: %#v", job) + } + } +} + +func TestListIncludesTerminalStatesAndStableReopen(t *testing.T) { + store, dir := openTestStore(t) + queued, err := store.Submit("client.a", sampleSubmission("req.list.cancelled")) + if err != nil { + t.Fatal(err) + } + cancelled, err := store.Cancel("client.a", queued.ID) + if err != nil || cancelled.State != StateCancelled { + t.Fatalf("cancel failed: %v %#v", err, cancelled) + } + + active, err := store.Submit("client.a", sampleSubmission("req.list.completed")) + if err != nil { + t.Fatal(err) + } + claim, err := store.Claim("client.a", active.ID, "worker.1") + if err != nil { + t.Fatal(err) + } + done, err := store.Complete(active.ID, claim.Token, Result{Text: "ok"}) + if err != nil || done.State != StateCompleted { + t.Fatalf("complete failed: %v %#v", err, done) + } + + before, err := store.List("client.a") + if err != nil { + t.Fatal(err) + } + if len(before) != 2 { + t.Fatalf("expected cancelled and completed jobs, got %#v", before) + } + states := map[string]bool{before[0].State: true, before[1].State: true} + if !states[StateCancelled] || !states[StateCompleted] { + t.Fatalf("missing terminal states: %#v", before) + } + + reopened, err := Open(dir, testKey) + if err != nil { + t.Fatal(err) + } + after, err := reopened.List("client.a") + if err != nil { + t.Fatal(err) + } + if len(after) != len(before) { + t.Fatalf("reopen changed list length: before=%d after=%d", len(before), len(after)) + } + for i := range before { + if after[i].ID != before[i].ID || after[i].State != before[i].State { + t.Fatalf("reopen changed order or identity: before=%#v after=%#v", before, after) + } + } +} + +func TestListOmitsSensitiveFieldsAndDoesNotAlias(t *testing.T) { + store, _ := openTestStore(t) + sub := sampleSubmission("req.list.sensitive") + sub.Messages[0].Content = "secret prompt body" + job, err := store.Submit("client.a", sub) + if err != nil { + t.Fatal(err) + } + claim, err := store.Claim("client.a", job.ID, "worker.1") + if err != nil || claim.Token == "" { + t.Fatalf("claim failed: %v %#v", err, claim) + } + completed, err := store.Complete(job.ID, claim.Token, Result{ + Text: "result text", + Artifacts: []Artifact{{URL: "https://example.com/a", Label: "a"}}, + }) + if err != nil { + t.Fatal(err) + } + + listed, err := store.List("client.a") + if err != nil || len(listed) != 1 { + t.Fatalf("list failed: %v %#v", err, listed) + } + payload, err := json.Marshal(listed) + if err != nil { + t.Fatal(err) + } + body := string(payload) + for _, forbidden := range []string{"token", "claim_token", "messages", "task", "secret prompt body", claim.Token} { + if strings.Contains(body, forbidden) { + t.Fatalf("serialized list leaked %q in %s", forbidden, body) + } + } + if listed[0].Result == nil || listed[0].Result.Text != completed.Result.Text { + t.Fatalf("expected public result, got %#v", listed[0]) + } + + listed[0].Summary = "mutated summary" + listed[0].Result.Text = "mutated result" + again, err := store.List("client.a") + if err != nil { + t.Fatal(err) + } + if again[0].Summary != completed.Summary || again[0].Result.Text != completed.Result.Text { + t.Fatalf("list results alias internal state: %#v", again[0]) + } +} From 8d15a46d82fdc4598b231c90dcfde0202f67f7bc Mon Sep 17 00:00:00 2001 From: Pyuyi <204892921+VeraPyuyi@users.noreply.github.com> Date: Mon, 19 Oct 2026 14:00:24 +0800 Subject: [PATCH 12/47] fix(dotevents): address minor refresh and persistence rereview findings Preserve callback verification failures across competing refreshes, while returning disabled/stale for paused or revoked state. Check persisted cache deadlines with legacy zero compatibility and bounded verification allowance. Protect newly created intermediate ancestors without changing existing parents. Compare failed-refresh TTL with the most recent successful refresh exactly. Keep NewDelivery and production public HTTPS transport unchanged. Evidence retained under: C:\Users\13678\AppData\Local\Temp\opencode\magpie-context-verification - dot-events-minor-red.json/.log: TestMinor reproduced error masking, invalid persisted cache acceptance, and inherited intermediate ACLs. - dot-events-minor-race.json/.log: go test -race ./internal/dotevents -count=1 PASS (package 5.367s, runner 8.32s). - dot-events-minor-vet.json/.log: go vet ./internal/dotevents PASS. - gofmt and git diff --cached --check clean. External report line 11 corrected and evidence appended. No full-suite run; controller owns integration checks. Commit contains only owned events files. --- docs/DOT_EVENTS.md | 6 +- internal/dotevents/cache_revision_test.go | 7 +- internal/dotevents/manager.go | 13 ++- internal/dotevents/minor_revision_test.go | 105 ++++++++++++++++++++++ internal/dotevents/paths.go | 3 + internal/dotevents/paths_windows_test.go | 36 ++++++++ internal/dotevents/state.go | 7 ++ 7 files changed, 169 insertions(+), 8 deletions(-) create mode 100644 internal/dotevents/minor_revision_test.go diff --git a/docs/DOT_EVENTS.md b/docs/DOT_EVENTS.md index c8871aa..502c7bd 100644 --- a/docs/DOT_EVENTS.md +++ b/docs/DOT_EVENTS.md @@ -2,11 +2,11 @@ `internal/dotevents` implements the subscription store and outbound webhook transport for MCP Events (2026-07-28), following the saved routing design and OpenAI's [MCP Events guide](https://developers.openai.com/plugins/build/mcp-events). Integration into MCP discovery and the runtime worker is a separate task. This package does not implement poll, stream, gap recovery, queue execution, or model readiness. -`Open(absoluteDir, key, delivery)` requires a caller-supplied 32-byte key. New stores start disabled. The authenticated owner must explicitly authorize activation through `SetEnabled(true)`; initialization never assumes a prior disable call. Existing stores retain their explicit activation state. A nil delivery or zero-value `Delivery` uses public HTTPS with normal certificate verification. The package has no production switch to permit private callbacks. Tests inject an unexported HTTP client with explicit local fixture routing and trusted fixture certificates. +`Open(absoluteDir, key, delivery)` requires a caller-supplied 32-byte key. New stores start disabled. The authenticated owner must explicitly authorize activation through `SetEnabled(true)`; initialization never assumes a prior disable call. Existing stores retain their explicit activation state. A nil delivery or zero-value `Delivery` uses public HTTPS with normal certificate verification. The package has no production switch to permit private callbacks. Tests inject an explicit HTTP client through `NewDelivery` with local fixture routing and trusted fixture certificates. The authenticated adapter supplies `principal`; the caller's `arguments.queue_id` must equal that principal. `List` advertises `dot.task.available` before a subscription exists. `Params` accepts `name`, `arguments:{queue_id}`, `delivery:{mode:"webhook",url,secret}`, optional null `cursor`, and raw `ttlMs`. Its JSON decoder rejects unknown/duplicate fields and null required objects. The adapter must enforce the request byte limit before decoding too. Omitted or null TTL means one hour, with an allowed range of one minute to one day. No non-expiring grant is supported. Secrets use `whsec_` plus standard base64 encoding of 24–64 bytes. URLs are limited to 2048 bytes and params to 16 KiB. -Subscription identity is SHA-256 over the principal, event, arguments, mode, and exact callback URL, excluding the secret and TTL. Refresh keeps that identity. The encrypted state is persisted before callback verification. Successful callback verification is cached by authenticated principal and exact callback URL for five minutes, bounded by subscription expiry. A cache hit refreshes without challenge I/O. Pause invalidates verification and cache; unsubscribe invalidates cache for that principal/URL. Every refresh gets a random generation; publishing the verification result requires the same generation after network I/O. After cache expiry, a failed challenge restores the previous accepted subscription only when the generation still matches, without extending its TTL or cache; it never restores a revoked subscription. Pause invalidates all callback verifications and generations, including across disable/re-enable. Unsubscribe selects the same identity without requiring a secret and is idempotent. The integration adapter must map the official unsubscribe request into this internal selection struct. +Subscription identity is SHA-256 over the principal, event, arguments, mode, and exact callback URL, excluding the secret and TTL. Refresh keeps that identity. The encrypted state is persisted before callback verification. Successful callback verification is cached by authenticated principal and exact callback URL for five minutes, bounded by subscription expiry. A cache hit refreshes without challenge I/O. Pause invalidates verification and cache; unsubscribe invalidates cache for that principal/URL. Every refresh gets a random generation; publishing the verification result requires the same generation after network I/O. After cache expiry, a failed challenge restores the previous accepted subscription only when the generation still matches, without extending its TTL or cache; it never restores a revoked subscription. Failed verification returns `disabled` while paused and `stale` after revocation; a competing refresh preserves the original verification error. Pause invalidates all callback verifications and generations, including across disable/re-enable. Unsubscribe selects the same identity without requiring a secret and is idempotent. The integration adapter must map the official unsubscribe request into this internal selection struct. Verification posts signed `{ "type":"verification", "challenge":"" }`. The unique nonce is used for one attempt bounded to ten seconds, below the five-minute verification maximum. A successful response must be 2xx with bounded JSON `{ "challenge":"" }`; comparison uses constant time. An event occurrence contains `eventId`, `name`, the caller-supplied `Notice.OccurredAt` serialized as RFC3339Nano (including its offset and fractional precision), `data:{queue_id,job_id,mode}`, and `cursor:null`. It never includes task prompts. Occurrence bytes are persisted on the first attempt and reused for retries. @@ -16,7 +16,7 @@ Every `Deliver` visits all matching verified subscriptions in sorted identity or Outbound transport disables environment proxies, redirects, connection reuse, and implicit DNS during socket dialing. Every new connection resolves with context, rejects the entire answer set if any address is non-public, then dials validated literal IPs while HTTP retains the original hostname for TLS SNI and certificate checks. Loopback, private, link-local, CGNAT, documentation, reserved, multicast, unspecified, IPv6 transition, and special-purpose ranges are rejected. Query strings are allowed and never copied into errors. Timeouts are at most ten seconds; response headers and verification bodies are limited to 16 KiB; event response bodies are discarded without reading; event bodies are capped at 256 KiB. -State uses AES-256-GCM, private directory/file permissions (protected current-user DACL on Windows), regular-file and hardlink checks, a five-second bounded flock, synced temporary files, and atomic replacement. Separate files are `events.state.de`, `dot_events.lock`, and `dot_events.initialized`. Every transaction validates the canonical local non-root path and each ancestor, rejects ancestor symlinks/reparse points, and restores directory and owned-file privacy before reading. Windows UNC/device/alternate-stream/reserved-name paths are rejected. Marker content must be exactly `1\n` and is read with a three-byte bound. Missing state after initialization and invalid ciphertext fail closed. This uses the repository's private-directory threat model; it does not claim to defeat an actively racing process with the same user privileges. No queue package private functions or files are used. +State uses AES-256-GCM, private directory/file permissions (protected current-user DACL on Windows), regular-file and hardlink checks, a five-second bounded flock, synced temporary files, and atomic replacement. Separate files are `events.state.de`, `dot_events.lock`, and `dot_events.initialized`. Every transaction validates the canonical local non-root path and each ancestor, rejects ancestor symlinks/reparse points, and restores directory and owned-file privacy before reading. Newly created intermediate ancestors receive private permissions; existing parents outside the store directory retain their permissions. Persisted nonzero verification deadlines must belong to a verified callback and fit the bounded verification lifetime; zero remains valid for legacy and invalidated caches. Windows UNC/device/alternate-stream/reserved-name paths are rejected. Marker content must be exactly `1\n` and is read with a three-byte bound. Missing state after initialization and invalid ciphertext fail closed. This uses the repository's private-directory threat model; it does not claim to defeat an actively racing process with the same user privileges. No queue package private functions or files are used. Errors are stable `dot_events_` values with optional retry duration and no callback URL, secret, response page, or filesystem path. `Status` distinguishes subscription and callback verification; `DotReady` is always false. The runtime must establish real readiness separately through authenticated claim/completion of a probe, and propagate pairing authorization changes into `SetEnabled` before allowing operations. diff --git a/internal/dotevents/cache_revision_test.go b/internal/dotevents/cache_revision_test.go index 9004d05..f534545 100644 --- a/internal/dotevents/cache_revision_test.go +++ b/internal/dotevents/cache_revision_test.go @@ -23,7 +23,7 @@ func TestCacheExpiryFailureAndRevocation(t *testing.T) { }) defer stop() p := testParams("alice") - first, e := m.Subscribe(context.Background(), "alice", p) + _, e := m.Subscribe(context.Background(), "alice", p) if e != nil { t.Fatal(e) } @@ -31,7 +31,8 @@ func TestCacheExpiryFailureAndRevocation(t *testing.T) { if e != nil { t.Fatal(e) } - if _, e = reopened.Subscribe(context.Background(), "alice", p); e != nil { + latest, e := reopened.Subscribe(context.Background(), "alice", p) + if e != nil { t.Fatal(e) } if challenges.Load() != 1 { @@ -52,7 +53,7 @@ func TestCacheExpiryFailureAndRevocation(t *testing.T) { t.Fatal("failed refresh destroyed accepted subscription") } _ = m.store.withLock(false, false, func(st *persistenceState) error { - if st.Subscriptions[first.ID].ExpiresAt.After(first.ExpiresAt.Add(time.Second)) { + if !st.Subscriptions[latest.ID].ExpiresAt.Equal(latest.ExpiresAt) { t.Error("failed refresh extended TTL") } return nil diff --git a/internal/dotevents/manager.go b/internal/dotevents/manager.go index f4a3a59..3697720 100644 --- a/internal/dotevents/manager.go +++ b/internal/dotevents/manager.go @@ -102,9 +102,17 @@ func (m *Manager) Subscribe(ctx context.Context, principal string, p Params) (Su // the generation race. Never extend its TTL or cache after failure. restoreErr := m.store.withLock(false, true, func(st *persistenceState) error { current := st.Subscriptions[id] - if !st.Enabled || current == nil || current.Generation != gen { + if !st.Enabled { + return errf("disabled") + } + if current == nil || !current.CallbackVerified && current.Generation == "" { return errf("stale") } + if current.Generation != gen { + // Another refresh owns this record. Preserve its state and the + // original callback failure unless authorization was revoked. + return nil + } if previous != nil && previous.CallbackVerified && previous.ExpiresAt.After(time.Now()) { st.Subscriptions[id] = previous } @@ -124,7 +132,8 @@ func (m *Manager) Subscribe(ctx context.Context, principal string, p Params) (Su return errf("stale") } s.CallbackVerified = true - s.VerifiedUntil = time.Now().Add(5 * time.Minute) + s.UpdatedAt = time.Now().UTC() + s.VerifiedUntil = s.UpdatedAt.Add(5 * time.Minute) return nil }) if e != nil { diff --git a/internal/dotevents/minor_revision_test.go b/internal/dotevents/minor_revision_test.go new file mode 100644 index 0000000..71c201d --- /dev/null +++ b/internal/dotevents/minor_revision_test.go @@ -0,0 +1,105 @@ +package dotevents + +import ( + "context" + "encoding/json" + "net/http" + "sync/atomic" + "testing" + "time" +) + +func TestMinorFailedVerificationRace(t *testing.T) { + for _, action := range []string{"pause", "reenable", "unsubscribe", "refresh"} { + t.Run(action, func(t *testing.T) { + entered, release := make(chan struct{}), make(chan struct{}) + var calls atomic.Int32 + m, _, stop, _ := withFixtureServerURL(t, func(w http.ResponseWriter, r *http.Request) { + if calls.Add(1) == 2 { + close(entered) + <-release + w.WriteHeader(http.StatusServiceUnavailable) + return + } + echoVerification(w, r) + }) + defer stop() + p := testParams("alice") + if _, e := m.Subscribe(context.Background(), "alice", p); e != nil { + t.Fatal(e) + } + if e := m.store.withLock(false, true, func(st *persistenceState) error { + st.Subscriptions[subscriptionID("alice", p)].VerifiedUntil = time.Now().Add(-time.Second) + return nil + }); e != nil { + t.Fatal(e) + } + done := make(chan error, 1) + go func() { _, e := m.Subscribe(context.Background(), "alice", p); done <- e }() + <-entered + var actionErr error + want := "stale" + switch action { + case "pause", "reenable": + actionErr = m.SetEnabled(false) + want = "disabled" + if action == "reenable" && actionErr == nil { + actionErr = m.SetEnabled(true) + want = "stale" + } + case "unsubscribe": + actionErr = m.Unsubscribe(context.Background(), "alice", p) + case "refresh": + _, actionErr = m.Subscribe(context.Background(), "alice", p) + want = "verification" + } + close(release) + e := <-done + if actionErr != nil { + t.Fatal(actionErr) + } + if e == nil || asEventsError(e).Code != want { + t.Fatalf("error = %v, want %s", e, want) + } + status, e := m.Status("alice", "alice") + if e != nil { + t.Fatal(e) + } + if status.CallbackVerified != (action == "refresh") { + t.Fatalf("verification after %s: %+v", action, status) + } + }) + } +} + +func TestMinorVerifiedUntilLoadValidation(t *testing.T) { + now := time.Now().UTC() + for _, tc := range []struct { + name string + until time.Time + verified, valid bool + }{ + {"legacy zero", time.Time{}, true, true}, + {"paused zero", time.Time{}, false, true}, + {"valid cache", now.Add(5 * time.Minute), true, true}, + {"expired cache", now.Add(-time.Minute), true, true}, + {"year zero", time.Date(0, 1, 1, 0, 0, 0, 0, time.UTC), true, false}, + {"excessive lifetime", now.Add(24 * time.Hour), true, false}, + {"unverified cache", now.Add(time.Minute), false, false}, + } { + t.Run(tc.name, func(t *testing.T) { + p := testParams("alice") + id := subscriptionID("alice", p) + st := newPersistenceState() + st.Subscriptions[id] = &subscriptionRecord{ID: id, Principal: "alice", Params: p, Secret: p.Delivery.Secret, CallbackVerified: tc.verified, VerifiedUntil: tc.until, CreatedAt: now, UpdatedAt: now, ExpiresAt: now.Add(time.Hour)} + data, e := json.Marshal(st) + if e != nil { + t.Fatal(e) + } + _, e = parsePersistenceState(data) + if (e == nil) != tc.valid { + t.Fatalf("load error = %v, valid = %v", e, tc.valid) + } + }) + } +} diff --git a/internal/dotevents/paths.go b/internal/dotevents/paths.go index ebc71e9..97e7ed6 100644 --- a/internal/dotevents/paths.go +++ b/internal/dotevents/paths.go @@ -51,6 +51,9 @@ func checkAncestors(dir string, create bool) error { return errf("unknown") } info, e = os.Lstat(p) + if e != nil || !info.IsDir() || rejectLink(info) != nil || privatePath(p, true) != nil { + return errf("unknown") + } } if e != nil || !info.IsDir() || rejectLink(info) != nil { return errf("unknown") diff --git a/internal/dotevents/paths_windows_test.go b/internal/dotevents/paths_windows_test.go index 8f162ef..1a00b47 100644 --- a/internal/dotevents/paths_windows_test.go +++ b/internal/dotevents/paths_windows_test.go @@ -29,6 +29,42 @@ func TestWindowsStoreBoundary(t *testing.T) { t.Fatal("junction ancestor accepted") } } +func TestMinorWindowsCreatedAncestors(t *testing.T) { + base := t.TempDir() + before, e := windows.GetNamedSecurityInfo(base, windows.SE_FILE_OBJECT, windows.DACL_SECURITY_INFORMATION) + if e != nil { + t.Fatal(e) + } + dir := filepath.Join(base, "created", "intermediate", "events") + if _, e = Open(dir, testKey, nil); e != nil { + t.Fatal(e) + } + after, e := windows.GetNamedSecurityInfo(base, windows.SE_FILE_OBJECT, windows.DACL_SECURITY_INFORMATION) + if e != nil { + t.Fatal(e) + } + if before.String() != after.String() { + t.Fatal("existing parent ACL changed") + } + user, e := windows.GetCurrentProcessToken().GetTokenUser() + if e != nil { + t.Fatal(e) + } + for p := dir; p != base; p = filepath.Dir(p) { + sd, e := windows.GetNamedSecurityInfo(p, windows.SE_FILE_OBJECT, windows.DACL_SECURITY_INFORMATION) + if e != nil { + t.Fatal(e) + } + acl, _, e := sd.DACL() + if e != nil { + t.Fatal(e) + } + if acl.AceCount != 1 || !strings.Contains(sd.String(), "D:P") || !strings.Contains(sd.String(), user.User.Sid.String()) { + t.Errorf("new ancestor lacks protected current-user ACL: %s %s", p, sd.String()) + } + } +} + func TestWindowsPrivacyEveryTransaction(t *testing.T) { m, dir := openTestManager(t) paths := []string{dir, filepath.Join(dir, stateFileName), filepath.Join(dir, markerName), filepath.Join(dir, lockName)} diff --git a/internal/dotevents/state.go b/internal/dotevents/state.go index 00c387f..e1e805d 100644 --- a/internal/dotevents/state.go +++ b/internal/dotevents/state.go @@ -80,6 +80,13 @@ func parsePersistenceState(data []byte) (*persistenceState, error) { return nil, errf("corrupt") } for id, sub := range st.Subscriptions { + if sub != nil && !sub.VerifiedUntil.IsZero() { + // Zero is valid for legacy records and invalidated caches. Older + // records stamped UpdatedAt before the bounded verification I/O. + if !sub.CallbackVerified || sub.VerifiedUntil.Year() < 1 || sub.VerifiedUntil.Year() > 9999 || sub.VerifiedUntil.After(sub.UpdatedAt.Add(5*time.Minute+maxHTTPTimeout)) { + return nil, errf("corrupt") + } + } if sub == nil || sub.ID != id || sub.ID != subscriptionID(sub.Principal, sub.Params) || validateID(sub.Principal) != nil || validateParams(sub.Principal, sub.Params) != nil || validateSecret(sub.Secret) != nil || sub.Secret != sub.Params.Delivery.Secret || (sub.PreviousSecret != "" && validateSecret(sub.PreviousSecret) != nil) { return nil, errf("corrupt") } From 219add2dad9b625188c69e3020170b14a451d7a1 Mon Sep 17 00:00:00 2001 From: Pyuyi <204892921+VeraPyuyi@users.noreply.github.com> Date: Mon, 19 Oct 2026 14:03:01 +0800 Subject: [PATCH 13/47] dotcontrol: operator bridge bootstrap, pairing files, and CLI hardening - Offline bridge enable writes dot_bridge_enabled via config.Save and requires restart before admin enable - Pair admin output redacted; probe --mode answer|task; jobs uses GET /dot-bridge/v1/jobs - Private pairing/state file helpers with ACL, link checks, and atomic replace - Document operator flow and controller integration gaps in research report --- cmd/dotcontrol/dot_bridge.go | 117 +++++++++++-- cmd/dotcontrol/dot_bridge_operator_test.go | 156 +++++++++++++++++ docs/DOT_BRIDGE.md | 10 +- internal/dotbridge/pairing.go | 184 +++++++++++++++++++- internal/dotbridge/pairing_operator_test.go | 45 +++++ internal/dotbridge/pairing_unix.go | 30 ++++ internal/dotbridge/pairing_windows.go | 73 ++++++++ research/dot-operator-setup-report.md | 32 ++++ 8 files changed, 616 insertions(+), 31 deletions(-) create mode 100644 cmd/dotcontrol/dot_bridge_operator_test.go create mode 100644 internal/dotbridge/pairing_operator_test.go create mode 100644 internal/dotbridge/pairing_unix.go create mode 100644 internal/dotbridge/pairing_windows.go create mode 100644 research/dot-operator-setup-report.md diff --git a/cmd/dotcontrol/dot_bridge.go b/cmd/dotcontrol/dot_bridge.go index c95d8bd..925657f 100644 --- a/cmd/dotcontrol/dot_bridge.go +++ b/cmd/dotcontrol/dot_bridge.go @@ -6,10 +6,13 @@ import ( "errors" "fmt" "io" + "net" "net/http" - "os" + "net/netip" "path/filepath" + "strconv" "strings" + "time" "github.com/dotcontrol/dotcontrol/internal/config" "github.com/dotcontrol/dotcontrol/internal/dotbridge" @@ -17,19 +20,35 @@ import ( "github.com/dotcontrol/dotcontrol/internal/vault" ) -func bridgeCLI(ctx context.Context, dir string, c config.Config, v vault.Vault, args []string) error { +func bridgeCLI(ctx context.Context, dir string, c config.Config, v vault.Store, args []string) error { if len(args) == 0 { return errors.New("bridge command required") } switch args[0] { case "enable": + if len(args) != 1 { + return errors.New("usage: bridge enable") + } + if !c.DotBridgeEnabled { + // Reload installed configuration instead of persisting a stale CLI snapshot. + current, err := config.Load(dir) + if err != nil { + return err + } + current.DotBridgeEnabled = true + if err := config.Save(dir, current); err != nil { + return err + } + fmt.Println(`{"configured":true,"restart_required":true,"next":"Restart the existing runtime, then run dotcontrol bridge enable"}`) + return nil + } return bridgePost(ctx, c, v, "/admin/bridge/enable", nil) case "disable": return bridgePost(ctx, c, v, "/admin/bridge/disable", nil) case "status": return bridgePost(ctx, c, v, "/admin/bridge/status", nil) case "pair": - return bridgePost(ctx, c, v, "/admin/bridge/pair", map[string]any{}) + return bridgePair(ctx, c, v) case "revoke-client": if len(args) < 2 { return errors.New("client_id required") @@ -39,12 +58,19 @@ func bridgeCLI(ctx context.Context, dir string, c config.Config, v vault.Vault, if len(args) < 2 { return errors.New("client_id required") } - return bridgePost(ctx, c, v, "/admin/bridge/probe", map[string]string{"client_id": args[1]}) + mode := "answer" + if len(args) != 2 { + if len(args) != 4 || args[2] != "--mode" || (args[3] != "answer" && args[3] != "task") { + return errors.New("usage: bridge probe [--mode answer|task]") + } + mode = args[3] + } + return bridgePost(ctx, c, v, "/admin/bridge/probe", map[string]string{"client_id": args[1], "mode": mode}) case "jobs": if len(args) < 2 { return errors.New("client_id required") } - return bridgeClientGet(ctx, dir, c.Listen, args[1], "/dot-bridge/v1/status") + return bridgeClientGet(ctx, dir, c.Listen, args[1], "/dot-bridge/v1/jobs") case "get": if len(args) < 3 { return errors.New("client_id and job_id required") @@ -60,12 +86,37 @@ func bridgeCLI(ctx context.Context, dir string, c config.Config, v vault.Vault, } } -func bridgePost(ctx context.Context, c config.Config, v vault.Vault, path string, body any) error { - token, err := v.Get("admin-token") +func bridgePair(ctx context.Context, c config.Config, v vault.Store) error { + data, err := bridgePostRaw(ctx, c, v, "/admin/bridge/pair", map[string]any{}) if err != nil { - return errors.New("local_credential_missing") + return err + } + var full map[string]json.RawMessage + if json.Unmarshal(data, &full) != nil { + return errors.New("invalid_agent_response") + } + out := make(map[string]string, 3) + for _, key := range []string{"pairing_file", "client_id", "queue_id"} { + if raw, ok := full[key]; ok { + var value string + if json.Unmarshal(raw, &value) == nil && value != "" { + out[key] = value + } + } + } + if out["pairing_file"] == "" { + return errors.New("invalid_agent_response") } - data, err := localapi.Client{Address: c.Listen, Token: token}.Post(ctx, path, body) + b, err := json.Marshal(out) + if err != nil { + return err + } + fmt.Println(string(b)) + return nil +} + +func bridgePost(ctx context.Context, c config.Config, v vault.Store, path string, body any) error { + data, err := bridgePostRaw(ctx, c, v, path, body) if err != nil { return err } @@ -73,7 +124,23 @@ func bridgePost(ctx context.Context, c config.Config, v vault.Vault, path string return nil } +func bridgePostRaw(ctx context.Context, c config.Config, v vault.Store, path string, body any) (json.RawMessage, error) { + if err := validateBridgeListen(c.Listen); err != nil { + return nil, err + } + ctx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + token, err := v.Get("admin-token") + if err != nil { + return nil, errors.New("local_credential_missing") + } + return localapi.Client{Address: c.Listen, Token: token}.Post(ctx, path, body) +} + func bridgeClientGet(ctx context.Context, dir, listen, clientID, path string) error { + if err := validateBridgeListen(listen); err != nil { + return err + } token, err := loadPairingToken(dir, clientID) if err != nil { return err @@ -82,6 +149,9 @@ func bridgeClientGet(ctx context.Context, dir, listen, clientID, path string) er } func bridgeClientPost(ctx context.Context, dir, listen, clientID, path string, body any) error { + if err := validateBridgeListen(listen); err != nil { + return err + } token, err := loadPairingToken(dir, clientID) if err != nil { return err @@ -90,6 +160,9 @@ func bridgeClientPost(ctx context.Context, dir, listen, clientID, path string, b } func bridgeClientRequest(ctx context.Context, listen, token, method, path string, body any) error { + if err := validateBridgeListen(listen); err != nil { + return err + } var reader io.Reader if body != nil { b, err := json.Marshal(body) @@ -106,7 +179,9 @@ func bridgeClientRequest(ctx context.Context, listen, token, method, path string if body != nil { req.Header.Set("Content-Type", "application/json") } - client := &http.Client{CheckRedirect: func(*http.Request, []*http.Request) error { return errors.New("redirect_refused") }} + transport := &http.Transport{Proxy: nil, DisableKeepAlives: true} + defer transport.CloseIdleConnections() + client := &http.Client{Transport: transport, Timeout: 10 * time.Second, CheckRedirect: func(*http.Request, []*http.Request) error { return errors.New("redirect_refused") }} res, err := client.Do(req) if err != nil { return err @@ -121,18 +196,26 @@ func bridgeClientRequest(ctx context.Context, listen, token, method, path string } func loadPairingToken(dir, clientID string) (string, error) { - path := filepath.Join(dir, "dot-bridge", "pairing-"+clientID+".json") - data, err := os.ReadFile(path) + file, err := dotbridge.ReadPairingFile(filepath.Join(dir, "dot-bridge"), clientID) if err != nil { - return "", errors.New("pairing_file_missing") - } - var file dotbridge.PairingFile - if json.Unmarshal(data, &file) != nil || file.Token == "" { - return "", errors.New("pairing_file_invalid") + return "", err } return file.Token, nil } +func validateBridgeListen(listen string) error { + host, port, err := net.SplitHostPort(listen) + if err != nil { + return errors.New("invalid_local_address") + } + ip, err := netip.ParseAddr(host) + n, portErr := strconv.Atoi(port) + if err != nil || !ip.IsLoopback() || ip.Zone() != "" || portErr != nil || n < 1 || n > 65535 { + return errors.New("invalid_local_address") + } + return nil +} + // Optional bridge failures preserve the existing runtime and its default catalog. func openRuntimeDotBridge(ctx context.Context, dir string, c config.Config, v vault.Store, paused, revoked func() bool) (*dotbridge.Service, error) { if e := dotbridge.EnsureStateKey(v); e != nil { diff --git a/cmd/dotcontrol/dot_bridge_operator_test.go b/cmd/dotcontrol/dot_bridge_operator_test.go new file mode 100644 index 0000000..8b1309c --- /dev/null +++ b/cmd/dotcontrol/dot_bridge_operator_test.go @@ -0,0 +1,156 @@ +package main + +import ( + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + + "github.com/dotcontrol/dotcontrol/internal/config" + "github.com/dotcontrol/dotcontrol/internal/dotbridge" +) + +func TestBridgeOperatorBootstrap(t *testing.T) { + dir := t.TempDir() + original := config.New("operator fixture", "fixture-driver") + if err := config.Save(dir, original); err != nil { + t.Fatal(err) + } + if err := bridgeCLI(context.Background(), dir, original, &unavailableBridgeVault{}, []string{"enable"}); err != nil { + t.Fatal(err) + } + got, err := config.Load(dir) + if err != nil { + t.Fatal(err) + } + original.DotBridgeEnabled = true + if !reflect.DeepEqual(got, original) { + t.Fatalf("configuration changed beyond bridge flag: %+v", got) + } + entries, _ := os.ReadDir(dir) + if len(entries) != 1 || entries[0].Name() != "config.json" { + t.Fatal("bootstrap created runtime state") + } +} + +func TestBridgeOperatorProbeModes(t *testing.T) { + var received map[string]string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/admin/bridge/probe" || r.Header.Get("X-DotControl-Token") != "fixture-admin" { + t.Error("unexpected admin request") + } + _ = json.NewDecoder(r.Body).Decode(&received) + _, _ = w.Write([]byte(`{"job_id":"fixture"}`)) + })) + defer srv.Close() + v := dotbridge.NewFakeVault() + _ = v.Set("admin-token", "fixture-admin") + c := config.Config{Listen: strings.TrimPrefix(srv.URL, "http://")} + for _, tc := range []struct { + args []string + mode string + }{ + {[]string{"probe", "client.fixture"}, "answer"}, + {[]string{"probe", "client.fixture", "--mode", "answer"}, "answer"}, + {[]string{"probe", "client.fixture", "--mode", "task"}, "task"}, + } { + if err := bridgeCLI(context.Background(), t.TempDir(), c, v, tc.args); err != nil { + t.Fatal(err) + } + if received["mode"] != tc.mode || received["client_id"] != "client.fixture" { + t.Fatalf("payload = %v", received) + } + } + for _, args := range [][]string{{"probe", "client", "--mode"}, {"probe", "client", "--mode", "bad"}, {"probe", "client", "--unknown"}, {"probe", "client", "extra"}, {"probe", "client", "--mode", "task", "extra"}} { + if err := bridgeCLI(context.Background(), t.TempDir(), c, &unavailableBridgeVault{}, args); err == nil || err.Error() == "local_credential_missing" { + t.Fatalf("args %v: %v", args, err) + } + } +} + +func TestBridgeOperatorRejectAddressBeforeCredential(t *testing.T) { + for _, address := range []string{"localhost:9317", "example.com:9317", "192.0.2.1:9317", "127.0.0.1:0"} { + err := bridgeClientGet(context.Background(), t.TempDir(), address, "client.fixture", "/dot-bridge/v1/status") + if err == nil || err.Error() != "invalid_local_address" { + t.Fatalf("%s: %v", address, err) + } + err = bridgePost(context.Background(), config.Config{Listen: address}, &unavailableBridgeVault{}, "/admin/bridge/status", nil) + if err == nil || err.Error() != "invalid_local_address" { + t.Fatalf("admin %s: %v", address, err) + } + } +} + +func TestBridgeOperatorPairRedactsToken(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(`{"pairing_file":"C:\\fixture\\pairing-client.fixture.json","client_id":"client.fixture","queue_id":"client.fixture","token":"secret-token"}`)) + })) + defer srv.Close() + v := dotbridge.NewFakeVault() + _ = v.Set("admin-token", "fixture-admin") + c := config.Config{Listen: strings.TrimPrefix(srv.URL, "http://")} + var printed strings.Builder + old := os.Stdout + r, w, _ := os.Pipe() + os.Stdout = w + err := bridgeCLI(context.Background(), t.TempDir(), c, v, []string{"pair"}) + _ = w.Close() + os.Stdout = old + _, _ = io.Copy(&printed, r) + if err != nil { + t.Fatal(err) + } + out := printed.String() + if strings.Contains(out, "secret-token") { + t.Fatalf("token leaked: %s", out) + } + if !strings.Contains(out, "pairing-client.fixture.json") || !strings.Contains(out, "client.fixture") { + t.Fatalf("metadata missing: %s", out) + } +} + +func TestBridgeOperatorJobsRequestPath(t *testing.T) { + var gotPath string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotPath = r.URL.Path + if r.Method != http.MethodGet { + t.Fatalf("method %s", r.Method) + } + if auth := r.Header.Get("Authorization"); auth != "Bearer fixture-pairing" { + t.Fatalf("auth %q", auth) + } + _, _ = w.Write([]byte(`{"version":1,"jobs":[]}`)) + })) + defer srv.Close() + dir := t.TempDir() + bridgeRoot := filepath.Join(dir, "dot-bridge") + pairingPath := filepath.Join(bridgeRoot, "pairing-client.fixture.json") + if err := os.MkdirAll(bridgeRoot, 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(pairingPath, []byte(`{"version":1,"client_id":"client.fixture","queue_id":"client.fixture","token":"fixture-pairing","listen":"127.0.0.1:9317"}`), 0600); err != nil { + t.Fatal(err) + } + if err := bridgeCLI(context.Background(), dir, config.Config{Listen: strings.TrimPrefix(srv.URL, "http://")}, &unavailableBridgeVault{}, []string{"jobs", "client.fixture"}); err != nil { + t.Fatal(err) + } + if gotPath != "/dot-bridge/v1/jobs" { + t.Fatalf("path %q", gotPath) + } +} + +func TestBridgeOperatorPairTraversal(t *testing.T) { + dir := t.TempDir() + _ = os.WriteFile(filepath.Join(dir, "pairing-target.json"), []byte(`{"token":"secret"}`), 0600) + for _, id := range []string{"../target", `..\target`, "x/y", "x:y", ".", ""} { + if _, err := loadPairingToken(dir, id); err == nil { + t.Fatalf("accepted %q", id) + } + } +} diff --git a/docs/DOT_BRIDGE.md b/docs/DOT_BRIDGE.md index 4258b3c..8e36f6f 100644 --- a/docs/DOT_BRIDGE.md +++ b/docs/DOT_BRIDGE.md @@ -1,16 +1,16 @@ # Dot bridge -The bridge is optional. Set `dot_bridge_enabled` in the installed local configuration and restart the runtime, then run `dotcontrol bridge enable`. The default installation retains 13 MCP tools. With the bridge available, MCP adds four task tools and three Events methods. A missing or unavailable vault key, or a bridge open failure, leaves the regular runtime and its 13 tools available; bridge admin operations report `bridge_unavailable`. +The bridge is optional. On a fresh install, run `dotcontrol bridge enable` once to set `dot_bridge_enabled` in the installed local configuration, restart the existing runtime, then run `dotcontrol bridge enable` again to turn the bridge on in the running service. The default installation retains 13 MCP tools. With the bridge available, MCP adds four task tools and three Events methods. A missing or unavailable vault key, or a bridge open failure, leaves the regular runtime and its 13 tools available; bridge admin operations report `bridge_unavailable`. -Run `dotcontrol bridge pair` to create a separate client credential and queue. The command prints a pairing file path and public IDs. Import the file locally into the client, then remove it when import is complete. Pairing locally authorizes the runtime's configured Dot consumer to act on that queue. Pairing grants no desktop, history, or terminal permission. +Run `dotcontrol bridge pair` to create a separate client credential and queue. The command prints only the pairing file path and public IDs (`client_id`, `queue_id`); it never prints the bearer token. Import the file locally into the client, then remove it when import is complete. Pairing locally authorizes the runtime's configured Dot consumer to act on that queue. Pairing grants no desktop, history, or terminal permission. Connect the private Dot plugin, refresh its metadata, and subscribe to `dot.task.available` using the returned queue ID. Events parameters are flat: `name`, `arguments`, `delivery`, and optional `cursor`/`ttlMs`. The SDK handles the Events RPC methods and protocol metadata checks. Only a successful modern `2026-07-28` `server/discover` response advertises `capabilities.events`. -Run `dotcontrol bridge probe ` explicitly to enqueue a nonce challenge. Through MCP, claim that job with `dot_task_claim`, then complete it with the exact requested nonce. The service checks the claim token, queue authorization, current verified callback subscription generation, and completed answer state. Expected nonce text is hashed in bridge policy; the challenge prompt remains in the encrypted queue. No manual readiness setter exists. +Run `dotcontrol bridge probe ` explicitly to enqueue a nonce challenge. Use `dotcontrol bridge probe --mode answer` for the default text readiness probe. `--mode task` is reserved for a future task verification probe and remains unavailable until an authorized task proof path is implemented; there is no manual readiness setter. Through MCP, claim that job with `dot_task_claim`, then complete it with the exact requested nonce. The service checks the claim token, queue authorization, current verified callback subscription generation, and completed answer state. Expected nonce text is hashed in bridge policy; the challenge prompt remains in the encrypted queue. -The paired Bearer credential accesses `/dot-bridge/v1/status`, `/jobs`, `/jobs/{id}`, `/requests/{request_id}`, and `/jobs/{id}/cancel` on the numeric loopback listener. No client-supplied identity selects a queue. Status exposes no prompt or credential. A successful text probe enables text readiness for at most ten minutes, bounded by subscription expiry. Pause, revoke, disable, renewal, and unsubscribe invalidate or suppress readiness. Runtime restart requires a fresh subscription and probe; live claim bindings are intentionally not restored. +The paired Bearer credential accesses `/dot-bridge/v1/status`, `GET /dot-bridge/v1/jobs` (metadata-only job list for the paired client, newest first; full `result` bodies require `GET /dot-bridge/v1/jobs/{id}`), `/jobs/{id}`, `/requests/{request_id}`, and `/jobs/{id}/cancel` on the numeric loopback listener. CLI bridge client requests use a 10s timeout, `Proxy: nil`, and refuse redirects. No client-supplied identity selects a queue. Status exposes no prompt or credential. A successful text probe enables text readiness for at most ten minutes, bounded by subscription expiry. Pause, revoke, disable, renewal, and unsubscribe invalidate or suppress readiness. Runtime restart requires a fresh subscription and probe; live claim bindings are intentionally not restored. -Regular submission requires current readiness. Task execution remains unavailable: `capabilities.tasks` is false and task submissions return `task_verification_required`. A text ping establishes no evidence about cloud task execution. An authorized task verification policy and work test are required before enabling that capability. +Regular submission requires current readiness. Task execution remains unavailable until task verification is proven: `capabilities.tasks` is false and task submissions return `task_verification_required`. A text ping establishes no evidence about cloud task execution. An authorized task verification policy and work test are required before enabling that capability. All MCP callers sharing the authenticated installation hop act as the same configured consumer. Queue IDs and worker IDs provide no authentication by themselves. A worker ID supplies retry identity and is combined with the trusted consumer identity. This proves a round trip through the configured source, not cryptographic model or personal identity. Fixture tests provide protocol evidence only, with no real account acceptance. diff --git a/internal/dotbridge/pairing.go b/internal/dotbridge/pairing.go index 2bf4079..fdfe2b8 100644 --- a/internal/dotbridge/pairing.go +++ b/internal/dotbridge/pairing.go @@ -2,26 +2,192 @@ package dotbridge import ( "encoding/json" + "errors" + "io" "os" "path/filepath" + "regexp" + "strings" + "unicode" "github.com/dotcontrol/dotcontrol/internal/vault" ) +var pairingID = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$`) +var errPrivateBridgeFile = errors.New("bridge_private_file_invalid") + +// The parent tree must be trusted local user/OS directories. Reject links in +// every component; this does not isolate against a malicious same-user process. +func pairingDirectory(root string, create bool) error { + if !filepath.IsAbs(root) || filepath.Clean(root) != root || strings.HasPrefix(filepath.VolumeName(root), `\\`) || strings.ContainsFunc(root, unicode.IsControl) || root == filepath.VolumeName(root)+string(filepath.Separator) || strings.Contains(strings.TrimPrefix(root, filepath.VolumeName(root)), ":") { + return errPrivateBridgeFile + } + for p := root; ; p = filepath.Dir(p) { + i, e := os.Lstat(p) + if e == nil { + if !i.IsDir() || pairingRejectLink(i) != nil { + return errPrivateBridgeFile + } + } else if !os.IsNotExist(e) { + return e + } + if filepath.Dir(p) == p { + break + } + } + if create { + if e := os.MkdirAll(root, 0700); e != nil { + return e + } + } + i, e := os.Lstat(root) + if e != nil { + return e + } + if !i.IsDir() || pairingRejectLink(i) != nil { + return errPrivateBridgeFile + } + return pairingPrivatePath(root, true) +} + +func pairingCheckFile(path string, missingOK bool) error { + i, e := os.Lstat(path) + if missingOK && os.IsNotExist(e) { + return nil + } + if e != nil { + return e + } + if !i.Mode().IsRegular() || pairingRejectLink(i) != nil { + return errPrivateBridgeFile + } + f, e := os.Open(path) + if e != nil { + return e + } + defer f.Close() + actual, e := f.Stat() + if e != nil { + return e + } + if !os.SameFile(i, actual) || pairingRejectHardlink(f, actual) != nil { + return errPrivateBridgeFile + } + return nil +} + +func writePrivateBridgeFile(root, name string, data []byte) error { + if e := pairingDirectory(root, true); e != nil { + return e + } + path := filepath.Join(root, name) + if e := pairingCheckFile(path, true); e != nil { + return e + } + f, e := os.CreateTemp(root, ".bridge-private-*") + if e != nil { + return e + } + tmp := f.Name() + defer os.Remove(tmp) + if e = pairingPrivatePath(tmp, false); e != nil { + f.Close() + return e + } + n, we := f.Write(data) + se := f.Sync() + ce := f.Close() + if we != nil { + return we + } + if se != nil { + return se + } + if ce != nil { + return ce + } + if n != len(data) { + return io.ErrShortWrite + } + if e = pairingCheckFile(path, true); e != nil { + return e + } + return pairingReplace(tmp, path) +} + +func readPrivateBridgeFile(root, name string) ([]byte, error) { + if e := pairingDirectory(root, false); e != nil { + return nil, e + } + path := filepath.Join(root, name) + if e := pairingCheckFile(path, false); e != nil { + return nil, e + } + f, e := os.Open(path) + if e != nil { + return nil, e + } + defer f.Close() + actual, e := f.Stat() + if e != nil { + return nil, e + } + current, e := os.Lstat(path) + if e != nil { + return nil, e + } + if !actual.Mode().IsRegular() || pairingRejectLink(current) != nil || !os.SameFile(actual, current) || pairingRejectHardlink(f, actual) != nil || actual.Size() > 1<<20 { + return nil, errPrivateBridgeFile + } + if e = pairingPrivatePath(path, false); e != nil { + return nil, e + } + return io.ReadAll(io.LimitReader(f, 1<<20)) +} + +// WritePairingFile writes a pairing credential to a private local file. +func WritePairingFile(root string, file PairingFile) (string, error) { + return writePairingFile(root, file) +} + func writePairingFile(root string, file PairingFile) (string, error) { - if err := os.MkdirAll(root, 0700); err != nil { - return "", err + if !pairingID.MatchString(file.ClientID) { + return "", errors.New("invalid_client_id") } name := "pairing-" + file.ClientID + ".json" - path := filepath.Join(root, name) - data, err := json.MarshalIndent(file, "", " ") - if err != nil { - return "", err + data, e := json.MarshalIndent(file, "", " ") + if e != nil { + return "", e + } + if e = writePrivateBridgeFile(root, name, data); e != nil { + return "", e + } + return filepath.Join(root, name), nil +} + +// ReadPairingFile validates the filename and private local file before loading a credential. +func ReadPairingFile(root, clientID string) (PairingFile, error) { + var file PairingFile + if !pairingID.MatchString(clientID) { + return file, errors.New("invalid_client_id") + } + data, e := readPrivateBridgeFile(root, "pairing-"+clientID+".json") + if e != nil { + return file, e } - if err := os.WriteFile(path, data, 0600); err != nil { - return "", err + if json.Unmarshal(data, &file) != nil || file.ClientID != clientID || file.Token == "" || file.Version != APIVersion { + return PairingFile{}, errors.New("pairing_file_invalid") } - return path, nil + return file, nil +} + +// Controller integration: replace service loadState/saveState raw file I/O with +// these helpers. Missing state retains os.IsNotExist semantics. +func securelyReadBridgeState(root string) ([]byte, error) { + return readPrivateBridgeFile(root, "bridge.state.json") +} +func securelyWriteBridgeState(root string, data []byte) error { + return writePrivateBridgeFile(root, "bridge.state.json", data) } func EnsureStateKey(v vault.Store) error { diff --git a/internal/dotbridge/pairing_operator_test.go b/internal/dotbridge/pairing_operator_test.go new file mode 100644 index 0000000..64cb13a --- /dev/null +++ b/internal/dotbridge/pairing_operator_test.go @@ -0,0 +1,45 @@ +package dotbridge + +import ( + "os" + "path/filepath" + "testing" +) + +func TestPairingOperatorRejectUnsafeIDs(t *testing.T) { + for _, id := range []string{"../escape", `..\escape`, "x/y", "x:y", ".", "", "x?y"} { + if _, err := writePairingFile(t.TempDir(), PairingFile{ClientID: id, Token: "fixture"}); err == nil { + t.Fatalf("accepted %q", id) + } + } +} + +func TestPairingOperatorRejectHardlink(t *testing.T) { + root := t.TempDir() + target := filepath.Join(root, "target") + if err := os.WriteFile(target, []byte("unchanged"), 0600); err != nil { + t.Fatal(err) + } + if err := os.Link(target, filepath.Join(root, "pairing-client.fixture.json")); err != nil { + t.Skip(err) + } + if _, err := writePairingFile(root, PairingFile{ClientID: "client.fixture", Token: "fixture"}); err == nil { + t.Fatal("accepted hardlink") + } + b, _ := os.ReadFile(target) + if string(b) != "unchanged" { + t.Fatal("modified hardlink target") + } +} + +func TestPairingOperatorAtomicRoundTrip(t *testing.T) { + root := filepath.Join(t.TempDir(), "bridge") + original := PairingFile{Version: APIVersion, ClientID: "client.fixture", QueueID: "client.fixture", Token: "fixture-secret", Listen: "127.0.0.1:9317"} + if _, err := writePairingFile(root, original); err != nil { + t.Fatal(err) + } + entries, _ := os.ReadDir(root) + if len(entries) != 1 { + t.Fatalf("temporary files left: %v", entries) + } +} diff --git a/internal/dotbridge/pairing_unix.go b/internal/dotbridge/pairing_unix.go new file mode 100644 index 0000000..d9964d7 --- /dev/null +++ b/internal/dotbridge/pairing_unix.go @@ -0,0 +1,30 @@ +//go:build !windows + +package dotbridge + +import ( + "os" + "syscall" +) + +func pairingPrivatePath(path string, directory bool) error { + mode := os.FileMode(0600) + if directory { + mode = 0700 + } + return os.Chmod(path, mode) +} +func pairingRejectLink(info os.FileInfo) error { + if info.Mode()&os.ModeSymlink != 0 { + return errPrivateBridgeFile + } + return nil +} +func pairingRejectHardlink(_ *os.File, info os.FileInfo) error { + stat, ok := info.Sys().(*syscall.Stat_t) + if !ok || stat.Nlink != 1 { + return errPrivateBridgeFile + } + return nil +} +func pairingReplace(from, to string) error { return os.Rename(from, to) } diff --git a/internal/dotbridge/pairing_windows.go b/internal/dotbridge/pairing_windows.go new file mode 100644 index 0000000..11cfe5a --- /dev/null +++ b/internal/dotbridge/pairing_windows.go @@ -0,0 +1,73 @@ +//go:build windows + +package dotbridge + +import ( + "os" + "syscall" + "time" + + "golang.org/x/sys/windows" +) + +func pairingPrivatePath(path string, directory bool) error { + user, err := windows.GetCurrentProcessToken().GetTokenUser() + if err != nil { + return err + } + inherit := "" + if directory { + inherit = "OICI" + } + sd, err := windows.SecurityDescriptorFromString("D:P(A;" + inherit + ";FA;;;" + user.User.Sid.String() + ")") + if err != nil { + return err + } + acl, _, err := sd.DACL() + if err != nil { + return err + } + return windows.SetNamedSecurityInfo(path, windows.SE_FILE_OBJECT, windows.DACL_SECURITY_INFORMATION|windows.PROTECTED_DACL_SECURITY_INFORMATION, nil, nil, acl, nil) +} + +func pairingRejectLink(info os.FileInfo) error { + if info.Mode()&os.ModeSymlink != 0 { + return errPrivateBridgeFile + } + if attr, ok := info.Sys().(*syscall.Win32FileAttributeData); ok && attr.FileAttributes&windows.FILE_ATTRIBUTE_REPARSE_POINT != 0 { + return errPrivateBridgeFile + } + return nil +} + +func pairingRejectHardlink(f *os.File, _ os.FileInfo) error { + var info windows.ByHandleFileInformation + if err := windows.GetFileInformationByHandle(windows.Handle(f.Fd()), &info); err != nil { + return err + } + if info.NumberOfLinks != 1 { + return errPrivateBridgeFile + } + return nil +} + +func pairingReplace(from, to string) error { + source, err := windows.UTF16PtrFromString(from) + if err != nil { + return err + } + target, err := windows.UTF16PtrFromString(to) + if err != nil { + return err + } + // Windows scanners/readers can temporarily deny replacement even after our + // own handles close. Retry only these transient errors, under the queue lock. + deadline := time.Now().Add(time.Second) + for { + e := windows.MoveFileEx(source, target, windows.MOVEFILE_REPLACE_EXISTING|windows.MOVEFILE_WRITE_THROUGH) + if e == nil || e != windows.ERROR_ACCESS_DENIED && e != windows.ERROR_SHARING_VIOLATION || time.Now().After(deadline) { + return e + } + time.Sleep(10 * time.Millisecond) + } +} diff --git a/research/dot-operator-setup-report.md b/research/dot-operator-setup-report.md new file mode 100644 index 0000000..0fca62c --- /dev/null +++ b/research/dot-operator-setup-report.md @@ -0,0 +1,32 @@ +# Dot operator setup report + +## Delivered in this worktree (operator-owned files) + +- `dotcontrol bridge enable` bootstraps offline: when `dot_bridge_enabled` is false, the CLI reloads installed `config.json`, sets only that flag via `config.Save`, prints `restart_required`, and does not call admin or start a duplicate runtime. +- `dotcontrol bridge pair` redacts admin responses to `pairing_file`, `client_id`, and `queue_id` only. +- `dotcontrol bridge probe [--mode answer|task]` sends `mode` in admin JSON (default `answer`). +- `dotcontrol bridge jobs ` calls `GET /dot-bridge/v1/jobs` (not `/status`). +- Bridge client HTTP: numeric loopback validation before credential load, 10s timeout, `Proxy: nil`, no redirects. +- Pairing and bridge state private file helpers in `internal/dotbridge/pairing*.go`: current-user ACL (Windows protected DACL), `0600`/`0700` Unix, ancestor trusted-parent checks, symlink/reparse/hardlink rejection, atomic replace, `client_id` validation. + +## Controller integration still required (not edited here) + +| Area | Caller | Action | +|------|--------|--------| +| Bridge state persistence | `service.loadState` / `service.saveState` | Replace raw `os.ReadFile`/`os.WriteFile` on `bridge.state.json` with `securelyReadBridgeState` / `securelyWriteBridgeState` from `internal/dotbridge/pairing.go`. Preserve `os.IsNotExist` → empty state on first load. | +| Admin pair response | `runtime.go` `/admin/bridge/pair` | Ensure JSON never includes `token` (today returns only path + IDs; `Service.Pair` still returns token to handler—do not echo it). | +| Admin probe mode | `runtime.go` `/admin/bridge/probe` | Decode `mode` from body; call `dotBridge.StartProbeMode(clientID, mode)` instead of `StartProbe` only. | +| Job list HTTP | `api.go` `handleJobs` | Route `GET /dot-bridge/v1/jobs` to scoped list (metadata-only `JobDTO`, `result` omitted); keep `POST` for submit. Depends on `QueueAPI.List` / `dotqueue` list export (commit `41f2748` area). | +| Windows DACL tests | `pairing_operator_test.go` or platform test | Add junction/reparse + protected DACL assertions mirroring `dotqueue/files_windows_test.go` once helpers are wired in CI. | + +## Pair API token risk + +`Service.Pair` returns `(PairingFile, path, err)` including the plaintext token for writing the local file. That is correct for in-process pairing. Risk is only if any admin or CLI layer serializes the full `PairingFile` to stdout or HTTP; operator CLI now filters pair output. + +## Tests run + +```text +go test ./cmd/dotcontrol -run TestBridgeOperator -count=1 +``` + +`internal/dotbridge` operator pairing tests require a clean build of sibling `probe.go` changes from the core worker (unused variable in flight at time of last run). From 8acb1765a665476d1914e5155a3a86feab6a3faf Mon Sep 17 00:00:00 2001 From: Pyuyi <204892921+VeraPyuyi@users.noreply.github.com> Date: Mon, 19 Oct 2026 14:09:24 +0800 Subject: [PATCH 14/47] feat(dotbridge): verify independent background task capability --- internal/dotbridge/probe.go | 31 ++- internal/dotbridge/service.go | 74 ++++--- internal/dotbridge/task_mode_test.go | 290 ++++++++++++++++++++++++++ internal/dotbridge/task_proof_test.go | 146 +++++++++++++ internal/dotbridge/types.go | 64 ++++-- internal/dotbridge/worker.go | 22 +- 6 files changed, 577 insertions(+), 50 deletions(-) create mode 100644 internal/dotbridge/task_mode_test.go create mode 100644 internal/dotbridge/task_proof_test.go diff --git a/internal/dotbridge/probe.go b/internal/dotbridge/probe.go index 0cb4404..ad261ef 100644 --- a/internal/dotbridge/probe.go +++ b/internal/dotbridge/probe.go @@ -18,6 +18,13 @@ type ProbeResult struct { } func (s *Service) StartProbe(clientID string) (ProbeResult, error) { + return s.StartProbeMode(clientID, dotqueue.ModeAnswer) +} + +func (s *Service) StartProbeMode(clientID, mode string) (ProbeResult, error) { + if mode != dotqueue.ModeAnswer && mode != dotqueue.ModeTask { + return ProbeResult{}, errors.New("invalid_probe_mode") + } s.mu.Lock() policy, ok := s.state.Clients[clientID] s.mu.Unlock() @@ -27,6 +34,10 @@ func (s *Service) StartProbe(clientID string) (ProbeResult, error) { if err := s.gate(); err != nil { return ProbeResult{}, err } + subscription, active := s.activeSubscription(policy.QueueID) + if !active { + return ProbeResult{}, errors.New("subscription_required") + } var nonce [16]byte if _, err := rand.Read(nonce[:]); err != nil { return ProbeResult{}, err @@ -42,6 +53,11 @@ func (s *Service) StartProbe(clientID string) (ProbeResult, error) { Content: fmt.Sprintf("Reply with exactly this nonce: %s", answer), }}, } + if mode == dotqueue.ModeTask { + sub.Mode = mode + sub.Messages = nil + sub.Task = fmt.Sprintf("In your own cloud workspace, create a temporary file with a unique name, write the nonce into it, read it back, then delete the file. After successful cleanup, return exactly the nonce and nothing else. Nonce: %s", answer) + } job, err := s.queue.Submit(clientID, sub) if err != nil { return ProbeResult{}, err @@ -55,10 +71,17 @@ func (s *Service) StartProbe(clientID string) (ProbeResult, error) { if !ok || policy.Revoked { return ProbeResult{}, errors.New("access_revoked") } - policy.ProbeJobID = job.ID - policy.ProbeAnswer = hashToken(answer) - policy.ProbeVerified = time.Time{} - policy.ProbeExpires = time.Time{} + if s.revoked != nil && s.revoked() { + return ProbeResult{}, errors.New("access_revoked") + } + if s.paused != nil && s.paused() { + return ProbeResult{}, errors.New("bridge_paused") + } + current := s.subscriptions[policy.QueueID] + if !s.state.Enabled || !time.Now().Before(current.Expires) || current.Generation != subscription.Generation || !current.Expires.Equal(subscription.Expires) { + return ProbeResult{}, errors.New("subscription_changed") + } + policy.setProof(mode, ProbeProof{JobID: job.ID, AnswerHash: hashToken(answer), Generation: subscription.Generation, SubscriptionExpires: subscription.Expires}) s.state.Clients[clientID] = policy if e := s.saveState(); e != nil { return ProbeResult{}, e diff --git a/internal/dotbridge/service.go b/internal/dotbridge/service.go index 829167d..f4b28a3 100644 --- a/internal/dotbridge/service.go +++ b/internal/dotbridge/service.go @@ -208,27 +208,45 @@ func (s *Service) StatusForClient(clientID string) (StatusDTO, error) { return s.statusDTO(policy), nil } func (s *Service) statusDTO(policy ClientPolicy) StatusDTO { - s.mu.Lock() - enabled := s.state.Enabled - current := s.state.Clients[policy.ClientID] - s.mu.Unlock() - sub, active := s.activeSubscription(policy.QueueID) + // Read policy and subscription together after external I/O. A renewal must + // never combine its subscription with a previously captured proof. eventStatus, e := s.events.Status(policy.QueueID, policy.QueueID) - ready := s.clientGate(policy.ClientID) == nil && active && e == nil && eventStatus.Subscribed && eventStatus.CallbackVerified && current.ProbeVerified.After(time.Time{}) && time.Now().Before(current.ProbeExpires) - out := StatusDTO{Version: APIVersion, Enabled: enabled, Paired: !current.Revoked, Code: "probe_required", Subscription: SubscriptionDTO{State: "inactive"}} + gateOK := s.clientGate(policy.ClientID) == nil + s.mu.Lock() + defer s.mu.Unlock() + current, exists := s.state.Clients[policy.ClientID] + sub := s.subscriptions[current.QueueID] + now := time.Now() + active := now.Before(sub.Expires) + out := StatusDTO{Version: APIVersion, Enabled: s.state.Enabled, Paired: exists && !current.Revoked, Code: "probe_required", Subscription: SubscriptionDTO{State: "inactive"}} if active { out.Subscription = SubscriptionDTO{State: "active", ExpiresAt: &sub.Expires} } - if ready { - expiry := current.ProbeExpires + if !gateOK || !s.state.Enabled || !exists || current.Revoked || !active || e != nil || !eventStatus.Subscribed || !eventStatus.CallbackVerified || (s.paused != nil && s.paused()) || (s.revoked != nil && s.revoked()) { + return out + } + for _, mode := range []string{dotqueue.ModeAnswer, dotqueue.ModeTask} { + proof := current.proof(mode) + if proof.Verified.IsZero() || !proof.Consumed || proof.Generation != sub.Generation || !now.Before(proof.Expires) || !proof.SubscriptionExpires.Equal(sub.Expires) { + continue + } + expiry := proof.Expires if sub.Expires.Before(expiry) { expiry = sub.Expires } - out.Ready = true - out.Code = "ready" - out.VerifiedAt = ¤t.ProbeVerified - out.ExpiresAt = &expiry - out.Capabilities = CapPolicy{Text: true} + if out.ExpiresAt == nil || expiry.Before(*out.ExpiresAt) { + out.ExpiresAt = &expiry + } + verified := proof.Verified + if out.VerifiedAt == nil || verified.Before(*out.VerifiedAt) { + out.VerifiedAt = &verified + } + if mode == dotqueue.ModeTask { + out.Capabilities.Tasks = true + } else { + out.Capabilities.Text = true + } + out.Ready, out.Code = true, "ready" } return out } @@ -326,8 +344,11 @@ func (s *Service) EventsSubscribe(ctx context.Context, principal string, params } old := s.subscriptions[queue] s.subscriptions[queue] = subscriptionProof{Expires: sub.ExpiresAt, Generation: old.Generation + 1} - p.ProbeVerified = time.Time{} + // Keep consumed timestamps for audit/replay protection. Generation binding + // invalidates both modes without erasing their first verification time. + p.Capabilities = CapPolicy{} p.ProbeExpires = time.Time{} + p.TaskProbe.Expires = time.Time{} s.state.Clients[queue] = p return sub, nil } @@ -366,8 +387,7 @@ func (s *Service) TaskStatus(principal, jobID, token string) (dotqueue.Job, erro } func (s *Service) loadState() error { - path := filepath.Join(s.root, "bridge.state.json") - data, err := os.ReadFile(path) + data, err := securelyReadBridgeState(s.root) if os.IsNotExist(err) { return nil } @@ -386,15 +406,11 @@ func (s *Service) loadState() error { } func (s *Service) saveState() error { - if err := os.MkdirAll(s.root, 0700); err != nil { - return err - } data, err := json.MarshalIndent(s.state, "", " ") if err != nil { return err } - path := filepath.Join(s.root, "bridge.state.json") - return os.WriteFile(path, data, 0600) + return securelyWriteBridgeState(s.root, data) } func (s *Service) submit(client ClientPolicy, req SubmitRequest) (JobDTO, int, error) { @@ -406,9 +422,10 @@ func (s *Service) submit(client ClientPolicy, req SubmitRequest) (JobDTO, int, e } status := s.statusDTO(client) if req.Mode == dotqueue.ModeTask { - return JobDTO{}, 0, errors.New("task_verification_required") - } - if !status.Ready { + if !status.Capabilities.Tasks { + return JobDTO{}, 0, errors.New("task_probe_required") + } + } else if !status.Capabilities.Text { return JobDTO{}, 0, errors.New("probe_required") } sub := dotqueue.Submission{ @@ -427,6 +444,10 @@ func (s *Service) submit(client ClientPolicy, req SubmitRequest) (JobDTO, int, e if e := s.clientGate(client.ClientID); e != nil { return JobDTO{}, 0, e } + postStatus := s.statusDTO(client) + if (req.Mode == dotqueue.ModeTask && !postStatus.Capabilities.Tasks) || (req.Mode != dotqueue.ModeTask && !postStatus.Capabilities.Text) { + return JobDTO{}, 0, errors.New("probe_required") + } dto := toJobDTO(job) dto.Accepted = true return dto, 202, nil @@ -499,6 +520,9 @@ func toJobDTO(job dotqueue.Job) JobDTO { } if job.Result != nil { dto.Result = &ResultDTO{Text: job.Result.Text, Cancelled: job.Result.Cancelled, ErrorCode: job.Result.ErrorCode} + for _, artifact := range job.Result.Artifacts { + dto.Result.Artifacts = append(dto.Result.Artifacts, ArtifactDTO{URL: artifact.URL, Label: artifact.Label}) + } } return dto } diff --git a/internal/dotbridge/task_mode_test.go b/internal/dotbridge/task_mode_test.go new file mode 100644 index 0000000..0fc40ea --- /dev/null +++ b/internal/dotbridge/task_mode_test.go @@ -0,0 +1,290 @@ +package dotbridge_test + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + "fmt" + "io" + "net" + "net/http" + "net/http/httptest" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/dotcontrol/dotcontrol/internal/dotbridge" + "github.com/dotcontrol/dotcontrol/internal/dotevents" + "github.com/dotcontrol/dotcontrol/internal/dotqueue" + "github.com/dotcontrol/dotcontrol/internal/localapi" + "github.com/dotcontrol/dotcontrol/internal/mcpserver" +) + +// These fixtures exercise the authenticated HTTP/MCP workflow, not a cloud computer. +type taskFixture struct { + s *dotbridge.Service + pair dotbridge.PairingFile + api *httptest.Server + rpc func(string, map[string]any) json.RawMessage + tool func(string, any) json.RawMessage + paused atomic.Bool +} + +func newTaskFixture(t *testing.T) *taskFixture { + t.Helper() + f := &taskFixture{} + callback := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var p map[string]any + _ = json.NewDecoder(r.Body).Decode(&p) + if challenge, ok := p["challenge"].(string); ok { + _ = json.NewEncoder(w).Encode(map[string]string{"challenge": challenge}) + return + } + w.WriteHeader(200) + })) + t.Cleanup(callback.Close) + tr := callback.Client().Transport.(*http.Transport).Clone() + tr.Proxy = nil + tr.TLSClientConfig.ServerName = "example.com" + tr.DialContext = func(ctx context.Context, network, address string) (net.Conn, error) { + return (&net.Dialer{}).DialContext(ctx, network, callback.Listener.Addr().String()) + } + delivery := dotevents.NewDelivery(&http.Client{Transport: tr, Timeout: 5 * time.Second, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }}) + v := dotbridge.NewFakeVault() + if err := dotbridge.EnsureStateKey(v); err != nil { + t.Fatal(err) + } + var err error + f.s, err = dotbridge.Open(context.Background(), dotbridge.Config{Root: filepath.Join(t.TempDir(), "bridge"), Listen: "127.0.0.1:9317", Vault: v, MCPQueue: "device.task-fixture", EventsDelivery: delivery, Paused: f.paused.Load}) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = f.s.Close() }) + if err = f.s.Enable(); err != nil { + t.Fatal(err) + } + f.pair, _, err = f.s.Pair("") + if err != nil { + t.Fatal(err) + } + h, err := mcpserver.HTTPWithBridge(backend{}, f.s) + if err != nil { + t.Fatal(err) + } + const hop = "fixture-mcp-credential-32-characters-long" + server := httptest.NewServer(localapi.Guard(hop, h)) + t.Cleanup(server.Close) + f.rpc = func(method string, params map[string]any) json.RawMessage { + t.Helper() + params["_meta"] = map[string]any{"io.modelcontextprotocol/protocolVersion": "2026-07-28", "io.modelcontextprotocol/clientCapabilities": map[string]any{}, "io.modelcontextprotocol/clientInfo": map[string]any{"name": "fixture", "version": "1"}} + b, _ := json.Marshal(map[string]any{"jsonrpc": "2.0", "id": 1, "method": method, "params": params}) + req, _ := http.NewRequest("POST", server.URL, bytes.NewReader(b)) + req.Header.Set("X-DotControl-Token", hop) + req.Header.Set("Mcp-Method", method) + req.Header.Set("MCP-Protocol-Version", "2026-07-28") + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Accept", "application/json, text/event-stream") + if method == "tools/call" { + req.Header.Set("Mcp-Name", params["name"].(string)) + } + res, err := server.Client().Do(req) + if err != nil { + t.Fatal(err) + } + defer res.Body.Close() + raw, _ := io.ReadAll(res.Body) + var env struct { + Result json.RawMessage + Error json.RawMessage + } + if json.Unmarshal(raw, &env) != nil || res.StatusCode != 200 || len(env.Error) > 0 { + t.Fatalf("%s: %d %s", method, res.StatusCode, raw) + } + return env.Result + } + f.tool = func(name string, args any) json.RawMessage { + t.Helper() + raw := f.rpc("tools/call", map[string]any{"name": name, "arguments": args}) + var result struct { + IsError bool + StructuredContent json.RawMessage + } + if json.Unmarshal(raw, &result) != nil || result.IsError { + t.Fatalf("%s: %s", name, raw) + } + return result.StructuredContent + } + f.api = httptest.NewServer(f.s.Handler()) + t.Cleanup(f.api.Close) + f.subscribe(t) + return f +} +func (f *taskFixture) subscribe(t *testing.T) { + t.Helper() + f.rpc("events/subscribe", map[string]any{"name": dotevents.EventName, "arguments": map[string]any{"queue_id": f.pair.QueueID}, "delivery": map[string]any{"mode": "webhook", "url": "https://callback.example/hook", "secret": "whsec_" + base64.StdEncoding.EncodeToString([]byte("01234567890123456789012345678901"))}}) +} +func (f *taskFixture) request(t *testing.T, method, path string, body any) (int, []byte) { + t.Helper() + b, _ := json.Marshal(body) + req, _ := http.NewRequest(method, f.api.URL+path, bytes.NewReader(b)) + req.Header.Set("Authorization", "Bearer "+f.pair.Token) + res, err := f.api.Client().Do(req) + if err != nil { + t.Fatal(err) + } + defer res.Body.Close() + raw, _ := io.ReadAll(res.Body) + return res.StatusCode, raw +} +func (f *taskFixture) status(t *testing.T, text, tasks bool) { + t.Helper() + code, raw := f.request(t, "GET", "/dot-bridge/v1/status", nil) + var s dotbridge.StatusDTO + if json.Unmarshal(raw, &s) != nil || code != 200 || s.Capabilities.Text != text || s.Capabilities.Tasks != tasks || s.Ready != (text || tasks) { + t.Fatalf("status want text=%v tasks=%v: %d %s", text, tasks, code, raw) + } +} +func (f *taskFixture) claim(t *testing.T, id string) dotqueue.Claim { + t.Helper() + var c dotqueue.Claim + if err := json.Unmarshal(f.tool("dot_task_claim", map[string]any{"queue_id": f.pair.QueueID, "job_id": id, "worker_id": "fixture." + id}), &c); err != nil { + t.Fatal(err) + } + return c +} +func (f *taskFixture) probe(t *testing.T, mode string) (dotqueue.Claim, string) { + t.Helper() + // A dynamic interface lets RED report the missing API as a behavioral failure. + starter, ok := any(f.s).(interface { + StartProbeMode(string, string) (dotbridge.ProbeResult, error) + }) + if !ok { + t.Fatal("missing mode-specific probe API") + } + p, err := starter.StartProbeMode(f.pair.ClientID, mode) + if err != nil { + t.Fatal(err) + } + c := f.claim(t, p.JobID) + if c.Submission.Mode != mode { + t.Fatalf("probe mode %q", c.Submission.Mode) + } + if mode == dotqueue.ModeAnswer { + return c, strings.TrimPrefix(c.Submission.Messages[0].Content, "Reply with exactly this nonce: ") + } + // Task prompt must request the reversible cloud operation and contain one nonce. + for _, word := range []string{"cloud workspace", "temporary file", "write", "read", "delete"} { + if !strings.Contains(c.Submission.Task, word) { + t.Fatalf("task prompt missing %q: %s", word, c.Submission.Task) + } + } + fields := strings.Fields(c.Submission.Task) + return c, fields[len(fields)-1] +} +func (f *taskFixture) complete(t *testing.T, c dotqueue.Claim, r dotqueue.Result) { + t.Helper() + f.tool("dot_task_complete", map[string]any{"job_id": c.Job.ID, "token": c.Token, "result": r}) +} +func TestTaskModeHTTPIndependentProofs(t *testing.T) { + f := newTaskFixture(t) + f.status(t, false, false) + task := map[string]any{"request_id": "normal.task", "session_id": "session", "mode": "task", "task": "fixture background work"} + code, raw := f.request(t, "POST", "/dot-bridge/v1/jobs", task) + if code == 202 { + t.Fatalf("task admitted without proof: %s", raw) + } + code, _ = f.request(t, "GET", "/dot-bridge/v1/requests/normal.task", nil) + if code == 200 { + t.Fatal("denied task persisted") + } + a, nonce := f.probe(t, "answer") + f.complete(t, a, dotqueue.Result{Text: nonce}) + f.status(t, true, false) + code, raw = f.request(t, "POST", "/dot-bridge/v1/jobs", task) + if code == 202 { + t.Fatalf("answer enabled task: %s", raw) + } + wrong, _ := f.probe(t, "task") + f.status(t, true, false) + f.complete(t, wrong, dotqueue.Result{Text: nonce}) + f.status(t, true, false) + c, taskNonce := f.probe(t, "task") + f.complete(t, c, dotqueue.Result{Text: taskNonce}) + f.status(t, true, true) + code, raw = f.request(t, "POST", "/dot-bridge/v1/jobs", task) + if code != 202 { + t.Fatalf("task not accepted: %d %s", code, raw) + } + var job dotbridge.JobDTO + _ = json.Unmarshal(raw, &job) + c = f.claim(t, job.JobID) + f.complete(t, c, dotqueue.Result{Text: "fixture result", Artifacts: []dotqueue.Artifact{{URL: "https://example.com/result", Label: "fixture artifact"}}}) + code, raw = f.request(t, "GET", "/dot-bridge/v1/jobs/"+job.JobID, nil) + if code != 200 || !bytes.Contains(raw, []byte("fixture artifact")) { + t.Fatalf("task result: %d %s", code, raw) + } + // A queued cancellation is terminal; a claimed cancellation remains intent + // until the consumer explicitly acknowledges it. + for _, claimed := range []bool{false, true} { + task["request_id"] = fmt.Sprintf("cancel.%v", claimed) + code, raw = f.request(t, "POST", "/dot-bridge/v1/jobs", task) + if code != 202 { + t.Fatalf("cancel submit: %d %s", code, raw) + } + if err := json.Unmarshal(raw, &job); err != nil { + t.Fatal(err) + } + if claimed { + c = f.claim(t, job.JobID) + } + code, raw = f.request(t, "POST", "/dot-bridge/v1/jobs/"+job.JobID+"/cancel", nil) + want := "cancelled" + if claimed { + want = "cancel_requested" + } + if code != 200 || !bytes.Contains(raw, []byte(`"state":"`+want+`"`)) { + t.Fatalf("cancel state: %d %s", code, raw) + } + if claimed { + f.complete(t, c, dotqueue.Result{Cancelled: true}) + code, raw = f.request(t, "GET", "/dot-bridge/v1/jobs/"+job.JobID, nil) + if code != 200 || !bytes.Contains(raw, []byte(`"state":"cancelled"`)) { + t.Fatalf("cancel acknowledgement: %d %s", code, raw) + } + } + } + a, nonce = f.probe(t, "answer") + f.status(t, false, true) + f.complete(t, a, dotqueue.Result{Text: nonce}) + f.status(t, true, true) + f.paused.Store(true) + f.status(t, false, false) + f.paused.Store(false) + f.subscribe(t) + f.status(t, false, false) +} +func TestTaskModeHTTPUnsuccessfulProbes(t *testing.T) { + for _, outcome := range []string{"cancelled", "failed", "wrong_nonce", "subscription_rotated"} { + t.Run(outcome, func(t *testing.T) { + f := newTaskFixture(t) + c, nonce := f.probe(t, "task") + r := dotqueue.Result{Text: nonce} + switch outcome { + case "cancelled": + f.request(t, "POST", "/dot-bridge/v1/jobs/"+c.Job.ID+"/cancel", nil) + r.Cancelled = true + case "failed": + r.ErrorCode = "fixture_failed" + case "wrong_nonce": + r.Text = "wrong" + case "subscription_rotated": + f.subscribe(t) + } + f.complete(t, c, r) + f.status(t, false, false) + }) + } +} diff --git a/internal/dotbridge/task_proof_test.go b/internal/dotbridge/task_proof_test.go new file mode 100644 index 0000000..e1849d5 --- /dev/null +++ b/internal/dotbridge/task_proof_test.go @@ -0,0 +1,146 @@ +package dotbridge + +import ( + "github.com/dotcontrol/dotcontrol/internal/dotevents" + "github.com/dotcontrol/dotcontrol/internal/dotqueue" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +func TestProbeCompletionReplayCannotExtendExpiry(t *testing.T) { + for _, mode := range []string{dotqueue.ModeAnswer, dotqueue.ModeTask} { + t.Run(mode, func(t *testing.T) { testProbeReplay(t, mode) }) + } +} + +func testProbeReplay(t *testing.T, mode string) { + s, _, id := openTestBridge(t) + p, err := s.StartProbeMode(id, mode) + if err != nil { + t.Fatal(err) + } + c, err := s.ClaimTask(id, p.JobID, "replay") + if err != nil { + t.Fatal(err) + } + nonce := "" + if mode == dotqueue.ModeTask { + fields := strings.Fields(c.Submission.Task) + nonce = fields[len(fields)-1] + } else { + nonce = strings.TrimPrefix(c.Submission.Messages[0].Content, "Reply with exactly this nonce: ") + } + r := dotqueue.Result{Text: nonce} + if _, err = s.CompleteTask(p.JobID, c.Token, r); err != nil { + t.Fatal(err) + } + past := time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC) + s.mu.Lock() + policy := s.state.Clients[id] + proof := policy.proof(mode) + proof.Verified = past + proof.Expires = past.Add(time.Minute) + policy.setProof(mode, proof) + s.state.Clients[id] = policy + if err = s.saveState(); err != nil { + t.Fatal(err) + } + // Reload durable metadata while retaining the authenticated in-memory binding. + if err = s.loadState(); err != nil { + t.Fatal(err) + } + s.mu.Unlock() + if _, err = s.CompleteTask(p.JobID, c.Token, r); err != nil { + t.Fatal(err) + } + status, _ := s.StatusForClient(id) + if status.Ready { + t.Fatal("identical completion replay extended expired proof") + } + s.mu.Lock() + defer s.mu.Unlock() + if !s.state.Clients[id].proof(mode).Verified.Equal(past) { + t.Fatal("first verification timestamp replaced") + } +} + +func TestBridgeStateServiceRejectsLinkedMetadata(t *testing.T) { + s, _, _ := openTestBridge(t) + path := filepath.Join(s.root, "bridge.state.json") + target := filepath.Join(t.TempDir(), "target") + if err := os.WriteFile(target, []byte(`{"version":1,"clients":{}}`), 0600); err != nil { + t.Fatal(err) + } + if err := os.Remove(path); err != nil { + t.Fatal(err) + } + if err := os.Link(target, path); err != nil { + t.Fatal(err) + } + if err := s.loadState(); err == nil { + t.Fatal("service read followed linked authorization metadata") + } + if err := s.saveState(); err == nil { + t.Fatal("service write followed linked authorization metadata") + } +} + +type wrongModeCompletion struct{ QueueAPI } + +func (q wrongModeCompletion) Complete(id, token string, result dotqueue.Result) (dotqueue.Job, error) { + job, err := q.QueueAPI.Complete(id, token, result) + job.Mode = dotqueue.ModeAnswer + return job, err +} +func TestTaskProbeRejectsCompletionModeMismatch(t *testing.T) { + s, _, id := openTestBridge(t) + p, err := s.StartProbeMode(id, dotqueue.ModeTask) + if err != nil { + t.Fatal(err) + } + c, err := s.ClaimTask(id, p.JobID, "mode") + if err != nil { + t.Fatal(err) + } + fields := strings.Fields(c.Submission.Task) + s.queue = wrongModeCompletion{QueueAPI: s.queue} + if _, err = s.CompleteTask(p.JobID, c.Token, dotqueue.Result{Text: fields[len(fields)-1]}); err != nil { + t.Fatal(err) + } + status, _ := s.StatusForClient(id) + if status.Ready { + t.Fatal("mismatched job mode proved capability") + } +} + +type renewingStatusEvents struct { + EventsAPI + renew func() +} + +func (e renewingStatusEvents) Status(a, b string) (dotevents.Status, error) { + e.renew() + return e.EventsAPI.Status(a, b) +} +func TestProbeStatusRenewalCannotMixGenerations(t *testing.T) { + s, _, id := openTestBridge(t) + p, err := s.StartProbe(id) + if err != nil { + t.Fatal(err) + } + c, err := s.ClaimTask(id, p.JobID, "status") + if err != nil { + t.Fatal(err) + } + if _, err = s.CompleteTask(p.JobID, c.Token, dotqueue.Result{Text: c.Submission.Messages[0].Content[len("Reply with exactly this nonce: "):]}); err != nil { + t.Fatal(err) + } + s.events = renewingStatusEvents{EventsAPI: s.events, renew: func() { VerifyTestSubscription(t, s, id) }} + status, _ := s.StatusForClient(id) + if status.Ready { + t.Fatal("status combined old proof with renewed subscription") + } +} diff --git a/internal/dotbridge/types.go b/internal/dotbridge/types.go index e91c086..ac44d07 100644 --- a/internal/dotbridge/types.go +++ b/internal/dotbridge/types.go @@ -9,16 +9,48 @@ const ( ) type ClientPolicy struct { - ClientID string `json:"client_id"` - QueueID string `json:"queue_id"` - TokenHash string `json:"token_hash"` - CreatedAt time.Time `json:"created_at"` - Revoked bool `json:"revoked"` - Capabilities CapPolicy `json:"capabilities"` - ProbeJobID string `json:"probe_job_id,omitempty"` - ProbeAnswer string `json:"probe_answer_hash,omitempty"` - ProbeVerified time.Time `json:"probe_verified_at,omitempty"` - ProbeExpires time.Time `json:"probe_expires_at,omitempty"` + ClientID string `json:"client_id"` + QueueID string `json:"queue_id"` + TokenHash string `json:"token_hash"` + CreatedAt time.Time `json:"created_at"` + Revoked bool `json:"revoked"` + Capabilities CapPolicy `json:"capabilities"` + ProbeJobID string `json:"probe_job_id,omitempty"` + ProbeAnswer string `json:"probe_answer_hash,omitempty"` + ProbeVerified time.Time `json:"probe_verified_at,omitempty"` + ProbeExpires time.Time `json:"probe_expires_at,omitempty"` + ProbeGeneration uint64 `json:"probe_generation,omitempty"` + ProbeSubscriptionExpires time.Time `json:"probe_subscription_expires_at,omitempty"` + ProbeConsumed bool `json:"probe_consumed,omitempty"` + TaskProbe ProbeProof `json:"task_probe,omitempty"` +} + +// ProbeProof is bounded private state for one mode's latest challenge. +// Consumed survives expiry and subscription rotation; retries cannot renew proof. +type ProbeProof struct { + JobID string `json:"job_id,omitempty"` + AnswerHash string `json:"answer_hash,omitempty"` + Verified time.Time `json:"verified_at,omitempty"` + Expires time.Time `json:"expires_at,omitempty"` + Generation uint64 `json:"generation,omitempty"` + SubscriptionExpires time.Time `json:"subscription_expires_at,omitempty"` + Consumed bool `json:"consumed,omitempty"` +} + +func (p ClientPolicy) proof(mode string) ProbeProof { + if mode == "task" { + return p.TaskProbe + } + return ProbeProof{JobID: p.ProbeJobID, AnswerHash: p.ProbeAnswer, Verified: p.ProbeVerified, Expires: p.ProbeExpires, Generation: p.ProbeGeneration, SubscriptionExpires: p.ProbeSubscriptionExpires, Consumed: p.ProbeConsumed} +} +func (p *ClientPolicy) setProof(mode string, proof ProbeProof) { + if mode == "task" { + p.TaskProbe = proof + return + } + p.ProbeJobID, p.ProbeAnswer = proof.JobID, proof.AnswerHash + p.ProbeVerified, p.ProbeExpires = proof.Verified, proof.Expires + p.ProbeGeneration, p.ProbeSubscriptionExpires, p.ProbeConsumed = proof.Generation, proof.SubscriptionExpires, proof.Consumed } type CapPolicy struct { @@ -66,9 +98,15 @@ type JobDTO struct { } type ResultDTO struct { - Text string `json:"text"` - Cancelled bool `json:"cancelled,omitempty"` - ErrorCode string `json:"error_code,omitempty"` + Text string `json:"text"` + Artifacts []ArtifactDTO `json:"artifacts,omitempty"` + Cancelled bool `json:"cancelled,omitempty"` + ErrorCode string `json:"error_code,omitempty"` +} + +type ArtifactDTO struct { + URL string `json:"url"` + Label string `json:"label"` } type SubmitRequest struct { diff --git a/internal/dotbridge/worker.go b/internal/dotbridge/worker.go index c3f6187..2f73801 100644 --- a/internal/dotbridge/worker.go +++ b/internal/dotbridge/worker.go @@ -142,17 +142,23 @@ func (s *Service) completeTask(jobID, token string, result dotqueue.Result) (dot sub := s.subscriptions[b.QueueID] active := time.Now().Before(sub.Expires) policy := s.state.Clients[b.QueueID] - if policy.Revoked { + if policy.Revoked || (s.revoked != nil && s.revoked()) { return dotqueue.Job{}, errors.New("access_revoked") } - if active && sub.Generation == b.Generation && policy.ProbeJobID == jobID && policy.ProbeAnswer == hashToken(result.Text) && job.State == dotqueue.StateCompleted && job.Mode == dotqueue.ModeAnswer { - now := time.Now().UTC() - policy.ProbeVerified = now - policy.ProbeExpires = now.Add(probeValidity) - if sub.Expires.Before(policy.ProbeExpires) { - policy.ProbeExpires = sub.Expires + if s.paused != nil && s.paused() { + return dotqueue.Job{}, errors.New("bridge_paused") + } + proof := policy.proof(job.Mode) + if s.state.Enabled && active && sub.Generation == b.Generation && proof.Generation == b.Generation && !sub.Expires.After(proof.SubscriptionExpires) && proof.JobID == jobID && proof.AnswerHash == hashToken(result.Text) && !proof.Consumed && proof.Verified.IsZero() && job.State == dotqueue.StateCompleted && (job.Mode == dotqueue.ModeAnswer || job.Mode == dotqueue.ModeTask) { + // Use the durable queue terminal timestamp, never the retry time. + proof.Verified = job.UpdatedAt + proof.Expires = job.UpdatedAt.Add(probeValidity) + proof.Consumed = true + proof.SubscriptionExpires = sub.Expires + if sub.Expires.Before(proof.Expires) { + proof.Expires = sub.Expires } - policy.Capabilities = CapPolicy{Text: true} + policy.setProof(job.Mode, proof) s.state.Clients[b.QueueID] = policy if e := s.saveState(); e != nil { return dotqueue.Job{}, e From 29d22bca227d119f2a7f53b6b850b13f8fd7ca10 Mon Sep 17 00:00:00 2001 From: Pyuyi <204892921+VeraPyuyi@users.noreply.github.com> Date: Mon, 19 Oct 2026 14:10:20 +0800 Subject: [PATCH 15/47] fix(dotbridge): forward operator probe mode to runtime Accept the explicit answer/task mode emitted by bridge probe, retaining answer as the default for older callers. Forward to StartProbeMode so the authorized admin path can create the independent background capability probe. Focused operator and HTTP mode tests passed via the isolated runner; live Dot acceptance remains outstanding. --- cmd/dotcontrol/runtime.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cmd/dotcontrol/runtime.go b/cmd/dotcontrol/runtime.go index b50d219..7390336 100644 --- a/cmd/dotcontrol/runtime.go +++ b/cmd/dotcontrol/runtime.go @@ -335,6 +335,7 @@ func serve(ctx context.Context, dir string, c config.Config, v vault.Vault, args case "/admin/bridge/probe": var input struct { ClientID string `json:"client_id"` + Mode string `json:"mode"` } r.Body = http.MaxBytesReader(w, r.Body, 1<<20) d := json.NewDecoder(r.Body) @@ -344,7 +345,10 @@ func serve(ctx context.Context, dir string, c config.Config, v vault.Vault, args } else if d.Decode(&input) != nil || input.ClientID == "" { e = errors.New("invalid_request") } else { - out, e = owner.dotBridge.StartProbe(input.ClientID) + if input.Mode == "" { + input.Mode = "answer" + } + out, e = owner.dotBridge.StartProbeMode(input.ClientID, input.Mode) } default: http.NotFound(w, r) From b22df486e0111982cc2f5bb981df0823d60407e9 Mon Sep 17 00:00:00 2001 From: Pyuyi <204892921+VeraPyuyi@users.noreply.github.com> Date: Mon, 19 Oct 2026 14:16:39 +0800 Subject: [PATCH 16/47] feat(dotqueue): expose capacity status and queue_full at limit --- docs/DOT_TASK_QUEUE.md | 4 +- internal/dotqueue/capacity.go | 26 +++ internal/dotqueue/capacity_test.go | 217 ++++++++++++++++++++++++++ internal/dotqueue/store.go | 2 +- research/dot-queue-capacity-report.md | 53 +++++++ 5 files changed, 299 insertions(+), 3 deletions(-) create mode 100644 internal/dotqueue/capacity.go create mode 100644 internal/dotqueue/capacity_test.go create mode 100644 research/dot-queue-capacity-report.md diff --git a/docs/DOT_TASK_QUEUE.md b/docs/DOT_TASK_QUEUE.md index ac24680..759f617 100644 --- a/docs/DOT_TASK_QUEUE.md +++ b/docs/DOT_TASK_QUEUE.md @@ -6,7 +6,7 @@ Call `Open(dir, key)` with a caller-managed 32-byte key and explicitly call `Ini Submission IDs (including client, queue, worker and optional session IDs) use `^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$`. Answer mode requires messages; task mode requires nonblank Task. The supported message roles are system, user and assistant. At most 32 messages are accepted. Combined Task and message content is at most 262144 UTF-8 bytes. Invalid UTF-8 is rejected before hashing. Canonical JSON uses fixed field order and empty arrays for nil message/artifact lists; SHA-256 identifies the complete submission/result. -Same client/request ID with identical content returns the original job; changed content conflicts. Queue ID is derived from client ID. Retained capacity is exactly 100 jobs, including terminal jobs, with no eviction or request-ID reuse. Acceptance and a single stable EventID are written atomically. Pending returns queued notices; retries and reopening preserve their IDs. Delivery/notification acknowledgment belongs to a later slice. +Same client/request ID with identical content returns the original job; changed content conflicts. Queue ID is derived from client ID. Retained capacity is exactly 100 jobs globally, including terminal jobs, with no eviction or request-ID reuse. When retained jobs already equal the limit, a new client/request pair fails with `queue_full` instead of `conflict`; idempotent retries for an existing request still succeed at capacity, while changed content for that request still conflicts. Hitting the retained maximum requires a planned archive or migration slice; this backend does not delete user data to make room. `Capacity()` returns only `Limit`, `Retained` and `Remaining` global counts (no client-identifying fields) while enabled, using the same enabled gate as reads. `ConfigStatus()` remains separate and reports only `Enabled`. Acceptance and a single stable EventID are written atomically. Pending returns queued notices; retries and reopening preserve their IDs. Delivery/notification acknowledgment belongs to a later slice. Claim binds a random 256-bit token to a job and worker. Same-worker retries return the original token; competing workers conflict. Worker mutations reject empty, malformed and incorrect tokens, and Complete requires a claimed lifecycle source. Public Job omits token and original submission; Claim contains both and must not be logged or forwarded as a public job response. @@ -22,6 +22,6 @@ Every transaction validates a canonical local absolute non-root directory and al The installed flock version exposes only pathname-based constructors and keeps its handle private. The queue therefore locks its own validated open handle using OS flock on Unix or LockFileEx on Windows, with a five-second acquisition budget, and checks that the locked handle still identifies the current lock path. This prevents the previous close-and-reopen gap. Read/validate/mutate/encrypt/temp write/sync/atomic replacement is one locked transaction. Windows uses MoveFileEx with WRITE_THROUGH; Unix uses rename plus directory sync. All temporary payloads are ciphertext. Private parents and ACLs protect against other OS principals; this design does **not** isolate a malicious same-user process that can replace directories, inspect process memory or obtain the caller's key. Path checks do not claim adversarial same-user race resistance. -Errors are sanitized `*dotqueue.Error`, with short `Code` values such as disabled, conflict, notfound, corrupt, claim_conflict and unknown; Error() adds `dot_queue_`. A nonnil error always means callers must discard any returned value. This API does not authenticate clientID/workerID arguments: future adapters must derive these identities from authenticated scope and keep Initialize/SetEnabled/Recover privileged. +Errors are sanitized `*dotqueue.Error`, with short `Code` values such as disabled, conflict, queue_full, notfound, corrupt, claim_conflict and unknown; Error() adds `dot_queue_`. Future HTTP adapters may map `queue_full` to HTTP 429. A nonnil error always means callers must discard any returned value. This API does not authenticate clientID/workerID arguments: future adapters must derive these identities from authenticated scope and keep Initialize/SetEnabled/Recover privileged. Tests use temporary directories and a fixed fake key. They include compile-safe behavioral regression cases, authenticated malformed-state fixtures, Windows ACL repair/reparse fixtures, actual child-process same-request submission/competing claim, literal size and capacity boundaries, persisted results and cancellation acknowledgment, encryption/tamper checks, and two-handle races. Run the package with the isolated runner and `go test ./internal/dotqueue -count=1 -v`; enable CGO and a compiler for `-race`. Exact retained RED/GREEN commands and review mappings are in the task report. No fixture establishes live account or routing readiness. diff --git a/internal/dotqueue/capacity.go b/internal/dotqueue/capacity.go new file mode 100644 index 0000000..6aa17a6 --- /dev/null +++ b/internal/dotqueue/capacity.go @@ -0,0 +1,26 @@ +package dotqueue + +// CapacityStatus reports global retained job counts for operator visibility. +// It intentionally omits client-identifying fields. +type CapacityStatus struct { + Limit int `json:"limit"` + Retained int `json:"retained"` + Remaining int `json:"remaining"` +} + +func (s *Store) Capacity() (CapacityStatus, error) { + var out CapacityStatus + err := s.withLock(false, false, func(st *persistenceState) error { + if !st.Enabled { + return errf("disabled") + } + retained := len(st.Jobs) + out = CapacityStatus{ + Limit: maxRetainedJobs, + Retained: retained, + Remaining: maxRetainedJobs - retained, + } + return nil + }) + return out, err +} diff --git a/internal/dotqueue/capacity_test.go b/internal/dotqueue/capacity_test.go new file mode 100644 index 0000000..3f9c026 --- /dev/null +++ b/internal/dotqueue/capacity_test.go @@ -0,0 +1,217 @@ +package dotqueue + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "testing" +) + +func fillStoreJobs(t *testing.T, store *Store, start int, count int, clientPrefix string) map[string]Job { + t.Helper() + jobs := make(map[string]Job, count) + for i := 0; i < count; i++ { + idx := start + i + clientID := fmt.Sprintf("%s.%d", clientPrefix, idx%10) + reqID := fmt.Sprintf("req.cap.%d", idx) + job, err := store.Submit(clientID, sampleSubmission(reqID)) + if err != nil { + t.Fatalf("fill %d: %v", idx, err) + } + jobs[reqID] = job + } + return jobs +} + +func TestCapacityStatusWhileFilling(t *testing.T) { + store, _ := openTestStore(t) + cap, err := store.Capacity() + if err != nil { + t.Fatal(err) + } + if cap.Limit != maxRetainedJobs || cap.Retained != 0 || cap.Remaining != maxRetainedJobs { + t.Fatalf("empty capacity %#v", cap) + } + + fillStoreJobs(t, store, 0, maxRetainedJobs, "client") + cap, err = store.Capacity() + if err != nil { + t.Fatal(err) + } + if cap.Retained != maxRetainedJobs || cap.Remaining != 0 { + t.Fatalf("full capacity %#v", cap) + } +} + +func TestCapacityQueueFullAcrossClients(t *testing.T) { + store, dir := openTestStore(t) + first := fillStoreJobs(t, store, 0, maxRetainedJobs, "client") + + overflow, err := store.Submit("client.overflow", sampleSubmission("req.overflow")) + if err == nil || overflow.ID != "" { + t.Fatalf("expected queue_full with empty job, got err=%v job=%#v", err, overflow) + } + if errorCode(err) != "queue_full" { + t.Fatalf("expected queue_full, got %v", err) + } + + cap, err := store.Capacity() + if err != nil || cap.Retained != maxRetainedJobs { + t.Fatalf("retained count after overflow: %v %#v", err, cap) + } + + again, err := store.Submit("client.0", sampleSubmission("req.cap.0")) + if err != nil || again.ID != first["req.cap.0"].ID { + t.Fatalf("idempotent submit at capacity failed: %v %#v", err, again) + } + + changed := sampleSubmission("req.cap.0") + changed.Messages[0].Content = "different payload" + _, err = store.Submit("client.0", changed) + if errorCode(err) != "conflict" { + t.Fatalf("expected conflict for changed request at capacity, got %v", err) + } + + reopened, err := Open(dir, testKey) + if err != nil { + t.Fatal(err) + } + cap2, err := reopened.Capacity() + if err != nil || cap2 != cap { + t.Fatalf("capacity changed on reopen: before=%#v after=%#v err=%v", cap, cap2, err) + } +} + +func TestCapacityDisabledDenied(t *testing.T) { + store, _ := openTestStore(t) + if err := store.SetEnabled(false); err != nil { + t.Fatal(err) + } + _, err := store.Capacity() + if errorCode(err) != "disabled" { + t.Fatalf("expected disabled, got %v", err) + } +} + +func TestCapacityEncryptedPersistence(t *testing.T) { + store, dir := openTestStore(t) + fillStoreJobs(t, store, 0, 3, "client") + before, err := store.Capacity() + if err != nil { + t.Fatal(err) + } + data, err := os.ReadFile(filepath.Join(dir, stateFileName)) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(data), "req.cap.") { + t.Fatal("request ids found in plaintext state file") + } + reopened, err := Open(dir, testKey) + if err != nil { + t.Fatal(err) + } + after, err := reopened.Capacity() + if err != nil || after != before { + t.Fatalf("reopen capacity mismatch: before=%#v after=%#v err=%v", before, after, err) + } +} + +// Child processes use only temporary fixture state and a fixed, non-secret test key. +func TestCapacityProcessHelper(t *testing.T) { + dir := os.Getenv("DOTQUEUE_CAPACITY_TEST_DIR") + if dir == "" { + return + } + store, err := Open(dir, testKey) + if err != nil { + t.Fatal(err) + } + switch os.Getenv("DOTQUEUE_CAPACITY_TEST_OP") { + case "submit_new": + _, e := store.Submit("client.child", sampleSubmission("req.child.overflow")) + if e != nil { + fmt.Println("SUBMIT=" + errorCode(e)) + } else { + fmt.Println("SUBMIT=ok") + } + case "capacity": + c, e := store.Capacity() + if e != nil { + fmt.Println("CAPACITY=" + errorCode(e)) + } else { + fmt.Printf("CAPACITY=%d,%d,%d\n", c.Limit, c.Retained, c.Remaining) + } + } +} + +func TestCapacityCrossProcessSubmitAtFull(t *testing.T) { + store, dir := openTestStore(t) + fillStoreJobs(t, store, 0, maxRetainedJobs, "client") + + cmd := exec.Command(os.Args[0], "-test.run=^TestCapacityProcessHelper$", "-test.v") + cmd.Env = append(os.Environ(), + "DOTQUEUE_CAPACITY_TEST_DIR="+dir, + "DOTQUEUE_CAPACITY_TEST_OP=submit_new", + ) + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("child failed: %v %s", err, out) + } + body := string(out) + if !strings.Contains(body, "SUBMIT=queue_full") { + t.Fatalf("expected child queue_full, got %s", body) + } + + cap, err := store.Capacity() + if err != nil || cap.Retained != maxRetainedJobs { + t.Fatalf("parent retained count changed: %v %#v", err, cap) + } +} + +func TestCapacityConcurrentIdempotentAtFull(t *testing.T) { + store, dir := openTestStore(t) + sub := sampleSubmission("req.idem.full") + first, err := store.Submit("client.a", sub) + if err != nil { + t.Fatal(err) + } + fillStoreJobs(t, store, 1, maxRetainedJobs-1, "client") + + other, err := Open(dir, testKey) + if err != nil { + t.Fatal(err) + } + + var wg sync.WaitGroup + start := make(chan struct{}) + var jobs [2]Job + var errs [2]error + for i, s := range []*Store{store, other} { + wg.Add(1) + go func(i int, s *Store) { + defer wg.Done() + <-start + jobs[i], errs[i] = s.Submit("client.a", sub) + }(i, s) + } + close(start) + wg.Wait() + + for i := 0; i < 2; i++ { + if errs[i] != nil { + t.Fatalf("idempotent submit %d failed: %v", i, errs[i]) + } + if jobs[i].ID != first.ID { + t.Fatalf("job %d changed id: %s vs %s", i, jobs[i].ID, first.ID) + } + } + + cap, err := store.Capacity() + if err != nil || cap.Retained != maxRetainedJobs { + t.Fatalf("capacity after concurrent idempotent submit %#v err=%v", cap, err) + } +} diff --git a/internal/dotqueue/store.go b/internal/dotqueue/store.go index 666d7b8..f9b60b4 100644 --- a/internal/dotqueue/store.go +++ b/internal/dotqueue/store.go @@ -69,7 +69,7 @@ func (s *Store) Submit(clientID string, sub Submission) (Job, error) { return nil } if len(st.Jobs) >= maxRetainedJobs { - return errf("conflict") + return errf("queue_full") } now := time.Now().UTC() jobID, err := newID("job") diff --git a/research/dot-queue-capacity-report.md b/research/dot-queue-capacity-report.md new file mode 100644 index 0000000..90cf838 --- /dev/null +++ b/research/dot-queue-capacity-report.md @@ -0,0 +1,53 @@ +# Dot queue capacity visibility slice report + +## Scope + +Owned changes only: + +- `internal/dotqueue/store.go` — new submissions at the global retained limit return `queue_full`. +- `internal/dotqueue/capacity.go` — `CapacityStatus` and `Store.Capacity()`. +- `internal/dotqueue/capacity_test.go` — behavioral and cross-process coverage. +- `docs/DOT_TASK_QUEUE.md` — operator semantics and error code documentation. + +Out of scope (controller/API): HTTP 429 mapping, DotControl UI wiring. + +## Behavior + +- Global retained limit remains `maxRetainedJobs` (100), counting terminal jobs; no eviction or request-index tombstone erasure. +- `Submit` idempotency is unchanged: same client/request with identical digest succeeds even when full; changed digest still returns `conflict`. +- New client/request pairs when `len(jobs) >= 100` return `queue_full` (not `conflict`). +- `Capacity()` uses `withLock(false, false)`, rejects when disabled (`dot_queue_disabled`), and returns `{Limit, Retained, Remaining}` only. +- `ConfigStatus()` remains the separate enabled gate without capacity fields. + +## Tests (RED → GREEN) + +| Test | Intent | +|------|--------| +| `TestCapacityStatusWhileFilling` | Limit/retained/remaining while empty and full | +| `TestCapacityQueueFullAcrossClients` | 101st `queue_full`, no extra job; idempotent + changed-conflict at full; reopen stable counts | +| `TestCapacityDisabledDenied` | Disabled gate on `Capacity()` | +| `TestCapacityEncryptedPersistence` | Counts survive reopen; no plaintext request ids on disk | +| `TestCapacityCrossProcessSubmitAtFull` | Child submit at full returns `queue_full`; parent count stays 100 | +| `TestCapacityConcurrentIdempotentAtFull` | Two-handle race for same request at capacity returns original job | + +## Verification commands + +Working directory: `D:\Applications\Opencode\data\share\worktree\bd2540\dot-task-event-bridge` + +```powershell +& "C:\Program Files\Go\bin\go.exe" test ./internal/dotqueue -count=1 -run TestCapacity -v +& "C:\Program Files\Go\bin\go.exe" test ./internal/dotqueue -count=1 -v +``` + +Optional race (requires CGO + C compiler): + +```powershell +$env:CGO_ENABLED=1 +& "C:\Program Files\Go\bin\go.exe" test -race ./internal/dotqueue -count=1 -run TestCapacityConcurrentIdempotentAtFull +``` + +## Follow-up + +- `internal/dotqueue/store_test.go` `TestMaxRetainedJobs` still expects `conflict`; update to `queue_full` when that file is in scope. +- API worker should map `queue_full` → HTTP 429 in the controller slice. +- Archive/migration slice required before operators can accept job 101+ without manual store reset. From dde312199209e769d2294224d29947047b1dd0c2 Mon Sep 17 00:00:00 2001 From: Pyuyi <204892921+VeraPyuyi@users.noreply.github.com> Date: Mon, 19 Oct 2026 14:18:00 +0800 Subject: [PATCH 17/47] test(dotqueue): align retained-job limit with queue_full Keep the existing retention regression aligned with the dedicated capacity error while leaving changed-request conflicts intact. Full internal/dotqueue race suite passed via isolated runner: 30.91 seconds; retained dotqueue-capacity-full-race.log. --- internal/dotqueue/store_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/dotqueue/store_test.go b/internal/dotqueue/store_test.go index 0ffb119..a3e5871 100644 --- a/internal/dotqueue/store_test.go +++ b/internal/dotqueue/store_test.go @@ -300,7 +300,7 @@ func TestMaxRetainedJobs(t *testing.T) { } } _, err := store.Submit("client.a", sampleSubmission("req.overflow")) - if errorCode(err) != "conflict" { - t.Fatalf("expected capacity conflict, got %v", err) + if errorCode(err) != "queue_full" { + t.Fatalf("expected queue_full at capacity, got %v", err) } } From 186ef952b0576372a4ad75ab66d8b0475a26afcd Mon Sep 17 00:00:00 2001 From: Pyuyi <204892921+VeraPyuyi@users.noreply.github.com> Date: Mon, 19 Oct 2026 14:18:30 +0800 Subject: [PATCH 18/47] fix(dotevents): separate persisted cache shape from clock expiry Accept authentic legacy verification timestamps without guessing a bound for lock/save/HTTP lifecycle time or backward wall-clock movement. Keep structural timestamp and verification checks on load. Bound every cache hit by the prior proof, source grant, new grant and now plus a shared five-minute cache TTL. Clamp fresh verification to subscription expiry and keep refresh CAS/revocation and public HTTPS protections intact. Deterministic tests cover legacy 10s/15s/25s spans, backward clocks, shorter TTL, source expiry, repeated refresh non-extension, reopen and invalid schema. Evidence: C:\Users\13678\AppData\Local\Temp\opencode\magpie-context-verification - dot-events-minor-clock-red.json/.log: behavioral failures reproduced. - dot-events-minor-clock-race.json/.log: go test -race ./internal/dotevents -count=1 PASS; package 6.188s, runner 9.49s. - dot-events-minor-clock-vet.json/.log: go vet ./internal/dotevents PASS. - gofmt and staged diff check clean. No full suite or subagents. Append rationale/evidence to external dot-events-task2-report.md. Correct the explicitly requested stale GCC statement in the List report using retained controller race evidence; no queue source/test changes. --- docs/DOT_EVENTS.md | 6 +- .../report/research/dot-queue-list-report.md | 2 +- internal/dotevents/cache_clock_test.go | 108 ++++++++++++++++++ internal/dotevents/manager.go | 31 ++++- internal/dotevents/minor_revision_test.go | 24 +++- internal/dotevents/state.go | 7 +- 6 files changed, 163 insertions(+), 15 deletions(-) create mode 100644 internal/dotevents/cache_clock_test.go diff --git a/docs/DOT_EVENTS.md b/docs/DOT_EVENTS.md index 502c7bd..edc383e 100644 --- a/docs/DOT_EVENTS.md +++ b/docs/DOT_EVENTS.md @@ -2,11 +2,11 @@ `internal/dotevents` implements the subscription store and outbound webhook transport for MCP Events (2026-07-28), following the saved routing design and OpenAI's [MCP Events guide](https://developers.openai.com/plugins/build/mcp-events). Integration into MCP discovery and the runtime worker is a separate task. This package does not implement poll, stream, gap recovery, queue execution, or model readiness. -`Open(absoluteDir, key, delivery)` requires a caller-supplied 32-byte key. New stores start disabled. The authenticated owner must explicitly authorize activation through `SetEnabled(true)`; initialization never assumes a prior disable call. Existing stores retain their explicit activation state. A nil delivery or zero-value `Delivery` uses public HTTPS with normal certificate verification. The package has no production switch to permit private callbacks. Tests inject an explicit HTTP client through `NewDelivery` with local fixture routing and trusted fixture certificates. +`Open(absoluteDir, key, delivery)` requires a caller-supplied 32-byte key. New stores start disabled. The authenticated owner must explicitly authorize activation through `SetEnabled(true)`; initialization never assumes a prior disable call. Existing stores retain their explicit activation state. A nil delivery or zero-value `Delivery` uses public HTTPS with normal certificate verification. The package has no production switch to permit private callbacks. Fixtures inject an explicit HTTP client, either within this package or through `NewDelivery`, with local fixture routing and trusted fixture certificates. The authenticated adapter supplies `principal`; the caller's `arguments.queue_id` must equal that principal. `List` advertises `dot.task.available` before a subscription exists. `Params` accepts `name`, `arguments:{queue_id}`, `delivery:{mode:"webhook",url,secret}`, optional null `cursor`, and raw `ttlMs`. Its JSON decoder rejects unknown/duplicate fields and null required objects. The adapter must enforce the request byte limit before decoding too. Omitted or null TTL means one hour, with an allowed range of one minute to one day. No non-expiring grant is supported. Secrets use `whsec_` plus standard base64 encoding of 24–64 bytes. URLs are limited to 2048 bytes and params to 16 KiB. -Subscription identity is SHA-256 over the principal, event, arguments, mode, and exact callback URL, excluding the secret and TTL. Refresh keeps that identity. The encrypted state is persisted before callback verification. Successful callback verification is cached by authenticated principal and exact callback URL for five minutes, bounded by subscription expiry. A cache hit refreshes without challenge I/O. Pause invalidates verification and cache; unsubscribe invalidates cache for that principal/URL. Every refresh gets a random generation; publishing the verification result requires the same generation after network I/O. After cache expiry, a failed challenge restores the previous accepted subscription only when the generation still matches, without extending its TTL or cache; it never restores a revoked subscription. Failed verification returns `disabled` while paused and `stale` after revocation; a competing refresh preserves the original verification error. Pause invalidates all callback verifications and generations, including across disable/re-enable. Unsubscribe selects the same identity without requiring a secret and is idempotent. The integration adapter must map the official unsubscribe request into this internal selection struct. +Subscription identity is SHA-256 over the principal, event, arguments, mode, and exact callback URL, excluding the secret and TTL. Refresh keeps that identity. The encrypted state is persisted before callback verification. Successful callback verification is cached by authenticated principal and exact callback URL for five minutes, bounded by subscription expiry. A cache hit refreshes without challenge I/O. Its saved deadline is the earliest of the prior deadline, prior subscription expiry, new subscription expiry, and the refresh clock plus five minutes. Repeated refreshes never extend the verification proof; a backward clock step can only shorten the saved deadline. Pause invalidates verification and cache; unsubscribe invalidates cache for that principal/URL. Every refresh gets a random generation; publishing the verification result requires the same generation after network I/O. After cache expiry, a failed challenge restores the previous accepted subscription only when the generation still matches, without extending its TTL or cache; it never restores a revoked subscription. Failed verification returns `disabled` while paused and `stale` after revocation; a competing refresh preserves the original verification error. Pause invalidates all callback verifications and generations, including across disable/re-enable. Unsubscribe selects the same identity without requiring a secret and is idempotent. The integration adapter must map the official unsubscribe request into this internal selection struct. Verification posts signed `{ "type":"verification", "challenge":"" }`. The unique nonce is used for one attempt bounded to ten seconds, below the five-minute verification maximum. A successful response must be 2xx with bounded JSON `{ "challenge":"" }`; comparison uses constant time. An event occurrence contains `eventId`, `name`, the caller-supplied `Notice.OccurredAt` serialized as RFC3339Nano (including its offset and fractional precision), `data:{queue_id,job_id,mode}`, and `cursor:null`. It never includes task prompts. Occurrence bytes are persisted on the first attempt and reused for retries. @@ -16,7 +16,7 @@ Every `Deliver` visits all matching verified subscriptions in sorted identity or Outbound transport disables environment proxies, redirects, connection reuse, and implicit DNS during socket dialing. Every new connection resolves with context, rejects the entire answer set if any address is non-public, then dials validated literal IPs while HTTP retains the original hostname for TLS SNI and certificate checks. Loopback, private, link-local, CGNAT, documentation, reserved, multicast, unspecified, IPv6 transition, and special-purpose ranges are rejected. Query strings are allowed and never copied into errors. Timeouts are at most ten seconds; response headers and verification bodies are limited to 16 KiB; event response bodies are discarded without reading; event bodies are capped at 256 KiB. -State uses AES-256-GCM, private directory/file permissions (protected current-user DACL on Windows), regular-file and hardlink checks, a five-second bounded flock, synced temporary files, and atomic replacement. Separate files are `events.state.de`, `dot_events.lock`, and `dot_events.initialized`. Every transaction validates the canonical local non-root path and each ancestor, rejects ancestor symlinks/reparse points, and restores directory and owned-file privacy before reading. Newly created intermediate ancestors receive private permissions; existing parents outside the store directory retain their permissions. Persisted nonzero verification deadlines must belong to a verified callback and fit the bounded verification lifetime; zero remains valid for legacy and invalidated caches. Windows UNC/device/alternate-stream/reserved-name paths are rejected. Marker content must be exactly `1\n` and is read with a three-byte bound. Missing state after initialization and invalid ciphertext fail closed. This uses the repository's private-directory threat model; it does not claim to defeat an actively racing process with the same user privileges. No queue package private functions or files are used. +State uses AES-256-GCM, private directory/file permissions (protected current-user DACL on Windows), regular-file and hardlink checks, a five-second bounded flock, synced temporary files, and atomic replacement. Separate files are `events.state.de`, `dot_events.lock`, and `dot_events.initialized`. Every transaction validates the canonical local non-root path and each ancestor, rejects ancestor symlinks/reparse points, and restores directory and owned-file privacy before reading. Newly created intermediate ancestors receive private permissions; existing parents outside the store directory retain their permissions. Persisted nonzero verification deadlines must be valid calendar timestamps belonging to a verified callback; zero remains valid for legacy and invalidated caches. Loading does not compare deadlines against `UpdatedAt`, `CreatedAt`, or the current clock: legacy timestamps preceded callback, lock and filesystem work, and wall clocks can move. These timestamps cannot prove elapsed verification time. Cache reuse separately enforces the five-minute and subscription-expiry bounds above, without increasing a legacy deadline or inventing a lifecycle allowance. Windows UNC/device/alternate-stream/reserved-name paths are rejected. Marker content must be exactly `1\n` and is read with a three-byte bound. Missing state after initialization and invalid ciphertext fail closed. This uses the repository's private-directory threat model; it does not claim to defeat an actively racing process with the same user privileges. No queue package private functions or files are used. Errors are stable `dot_events_` values with optional retry duration and no callback URL, secret, response page, or filesystem path. `Status` distinguishes subscription and callback verification; `DotReady` is always false. The runtime must establish real readiness separately through authenticated claim/completion of a probe, and propagate pairing authorization changes into `SetEnabled` before allowing operations. diff --git a/evidence/report/research/dot-queue-list-report.md b/evidence/report/research/dot-queue-list-report.md index 3c65f5f..19ec9d4 100644 --- a/evidence/report/research/dot-queue-list-report.md +++ b/evidence/report/research/dot-queue-list-report.md @@ -34,7 +34,7 @@ Working directory: `D:\Applications\Opencode\data\share\worktree\bd2540\dot-task & "C:\Program Files\Go\bin\go.exe" test -race ./internal/dotqueue -count=1 -run TestList ``` -Results: all `TestList*` cases passed on 2026-10-19. `-race` was attempted with `CGO_ENABLED=1` but the environment has no C compiler (`gcc` not found); rerun `-race` where CGO and a compiler are available. +Results: all `TestList*` cases passed on 2026-10-19. The initial worker race attempt could not locate gcc. The controller subsequently ran `C:\Program Files\Go\bin\go.exe test -race -count=1 -timeout 1m ./internal/dotqueue -run ^TestList` successfully (exit 0, runner 4.23s). Retained evidence: `C:\Users\13678\AppData\Local\Temp\opencode\magpie-context-verification\dotqueue-list-controller-race.json` and `.log`. GCC is available at `C:\msys64\mingw64\bin\gcc.exe`; the external runner configures that compiler for race checks. ## Follow-up diff --git a/internal/dotevents/cache_clock_test.go b/internal/dotevents/cache_clock_test.go new file mode 100644 index 0000000..db9506d --- /dev/null +++ b/internal/dotevents/cache_clock_test.go @@ -0,0 +1,108 @@ +package dotevents + +import ( + "context" + "net/http" + "sync/atomic" + "testing" + "time" +) + +func TestCacheClockBounds(t *testing.T) { + base := time.Date(2026, 10, 19, 12, 0, 0, 0, time.UTC) + for _, tc := range []struct { + name string + until, oldExpiry, now, want time.Time + ttl string + }{ + {"backward clock", base.Add(5 * time.Minute), base.Add(time.Hour), base.Add(-time.Hour), base.Add(-55 * time.Minute), "3600000"}, + {"shorter refresh", base.Add(5 * time.Minute), base.Add(time.Hour), base, base.Add(time.Minute), "60000"}, + {"source expiry", base.Add(5 * time.Minute), base.Add(2 * time.Minute), base, base.Add(2 * time.Minute), "3600000"}, + {"no proof extension", base.Add(time.Minute), base.Add(time.Hour), base, base.Add(time.Minute), "3600000"}, + } { + t.Run(tc.name, func(t *testing.T) { + var calls atomic.Int32 + m, dir, stop, _ := withFixtureServerURL(t, func(w http.ResponseWriter, r *http.Request) { calls.Add(1); echoVerification(w, r) }) + defer stop() + p := testParams("alice") + id := subscriptionID("alice", p) + if e := m.store.withLock(false, true, func(st *persistenceState) error { + st.Subscriptions[id] = &subscriptionRecord{ID: id, Generation: "legacy", Principal: "alice", Params: p, Secret: p.Delivery.Secret, CallbackVerified: true, CreatedAt: base, UpdatedAt: base, ExpiresAt: tc.oldExpiry, VerifiedUntil: tc.until} + return nil + }); e != nil { + t.Fatal(e) + } + p.TTL = []byte(tc.ttl) + if _, e := m.subscribe(context.Background(), "alice", p, func() time.Time { return tc.now }); e != nil { + t.Fatal(e) + } + reopened, e := Open(dir, testKey, m.delivery) + if e != nil { + t.Fatal(e) + } + if e = reopened.store.withLock(false, false, func(st *persistenceState) error { + s := st.Subscriptions[id] + if !s.VerifiedUntil.Equal(tc.want) { + t.Errorf("deadline %s, want %s", s.VerifiedUntil, tc.want) + } + if !s.CreatedAt.Equal(base) || s.Generation == "legacy" { + t.Error("refresh lost creation time or generation") + } + return nil + }); e != nil { + t.Fatal(e) + } + if _, e := reopened.subscribe(context.Background(), "alice", p, func() time.Time { return tc.now.Add(30 * time.Second) }); e != nil { + t.Fatal(e) + } + if e := reopened.store.withLock(false, false, func(st *persistenceState) error { + if !st.Subscriptions[id].VerifiedUntil.Equal(tc.want) { + t.Error("repeated refresh extended verification proof") + } + return nil + }); e != nil { + t.Fatal(e) + } + if calls.Load() != 0 { + t.Fatal("cache hit performed IO") + } + }) + } +} + +func TestSlowVerificationClock(t *testing.T) { + base := time.Date(2026, 10, 19, 12, 0, 0, 0, time.UTC) + for _, elapsed := range []time.Duration{10 * time.Second, 15 * time.Second, 25 * time.Second} { + t.Run(elapsed.String(), func(t *testing.T) { + current := base + m, dir, stop, _ := withFixtureServerURL(t, func(w http.ResponseWriter, r *http.Request) { echoVerification(w, r) }) + defer stop() + // Model callback plus both lock/save spans without waiting on real time. + clockCalls := 0 + clock := func() time.Time { + clockCalls++ + if clockCalls > 1 { + current = base.Add(elapsed) + } + return current + } + p := testParams("alice") + if _, e := m.subscribe(context.Background(), "alice", p, clock); e != nil { + t.Fatal(e) + } + reopened, e := Open(dir, testKey, m.delivery) + if e != nil { + t.Fatal(e) + } + if e = reopened.store.withLock(false, false, func(st *persistenceState) error { + s := st.Subscriptions[subscriptionID("alice", p)] + if !s.VerifiedUntil.Equal(base.Add(elapsed + 5*time.Minute)) { + t.Errorf("deadline %s", s.VerifiedUntil) + } + return nil + }); e != nil { + t.Fatal(e) + } + }) + } +} diff --git a/internal/dotevents/manager.go b/internal/dotevents/manager.go index 3697720..8e92435 100644 --- a/internal/dotevents/manager.go +++ b/internal/dotevents/manager.go @@ -6,6 +6,8 @@ import ( "time" ) +const verifiedCacheTTL = 5 * time.Minute + type Manager struct { store *store delivery *Delivery @@ -39,6 +41,10 @@ func (m *Manager) SetEnabled(enabled bool) error { }) } func (m *Manager) Subscribe(ctx context.Context, principal string, p Params) (Subscription, error) { + return m.subscribe(ctx, principal, p, time.Now) +} + +func (m *Manager) subscribe(ctx context.Context, principal string, p Params, clock func() time.Time) (Subscription, error) { if e := validateParams(principal, p); e != nil { return Subscription{}, e } @@ -48,7 +54,7 @@ func (m *Manager) Subscribe(ctx context.Context, principal string, p Params) (Su if e != nil { return Subscription{}, e } - now := time.Now().UTC() + now := clock().UTC() var previous *subscriptionRecord var cached bool rec := &subscriptionRecord{ID: id, Generation: gen, Principal: principal, Params: p, Secret: p.Delivery.Secret, ExpiresAt: now.Add(time.Duration(ttl) * time.Millisecond), CreatedAt: now, UpdatedAt: now} @@ -72,7 +78,7 @@ func (m *Manager) Subscribe(ctx context.Context, principal string, p Params) (Su if s.Principal == principal && s.Params.Delivery.URL == p.Delivery.URL && s.CallbackVerified && s.VerifiedUntil.After(now) && s.ExpiresAt.After(now) { cached = true rec.CallbackVerified = true - rec.VerifiedUntil = s.VerifiedUntil + rec.VerifiedUntil = boundedVerificationDeadline(s.VerifiedUntil, s.ExpiresAt, rec.ExpiresAt, now) break } } @@ -113,7 +119,7 @@ func (m *Manager) Subscribe(ctx context.Context, principal string, p Params) (Su // original callback failure unless authorization was revoked. return nil } - if previous != nil && previous.CallbackVerified && previous.ExpiresAt.After(time.Now()) { + if previous != nil && previous.CallbackVerified && previous.ExpiresAt.After(clock()) { st.Subscriptions[id] = previous } return nil @@ -128,12 +134,12 @@ func (m *Manager) Subscribe(ctx context.Context, principal string, p Params) (Su return errf("disabled") } s := st.Subscriptions[id] - if s == nil || s.Generation != gen || !s.ExpiresAt.After(time.Now()) { + if s == nil || s.Generation != gen || !s.ExpiresAt.After(clock()) { return errf("stale") } s.CallbackVerified = true - s.UpdatedAt = time.Now().UTC() - s.VerifiedUntil = s.UpdatedAt.Add(5 * time.Minute) + s.UpdatedAt = clock().UTC() + s.VerifiedUntil = boundedVerificationDeadline(s.UpdatedAt.Add(verifiedCacheTTL), s.ExpiresAt, s.ExpiresAt, s.UpdatedAt) return nil }) if e != nil { @@ -141,6 +147,19 @@ func (m *Manager) Subscribe(ctx context.Context, principal string, p Params) (Su } return Subscription{ID: id, ExpiresAt: rec.ExpiresAt, CallbackVerified: true}, nil } + +// Never extend an accepted verification proof on refresh, even after a +// backward clock step. Legacy deadlines need no guessed lock/IO allowance: +// each reuse is capped by both subscription grants and the current cache TTL. +func boundedVerificationDeadline(until, sourceExpiry, expiry, now time.Time) time.Time { + for _, bound := range []time.Time{sourceExpiry, expiry, now.Add(verifiedCacheTTL)} { + if bound.Before(until) { + until = bound + } + } + return until +} + func (m *Manager) Unsubscribe(ctx context.Context, principal string, p Params) error { if ctx.Err() != nil { return errf("cancelled") diff --git a/internal/dotevents/minor_revision_test.go b/internal/dotevents/minor_revision_test.go index 71c201d..7d8ae34 100644 --- a/internal/dotevents/minor_revision_test.go +++ b/internal/dotevents/minor_revision_test.go @@ -73,7 +73,7 @@ func TestMinorFailedVerificationRace(t *testing.T) { } func TestMinorVerifiedUntilLoadValidation(t *testing.T) { - now := time.Now().UTC() + now := time.Date(2026, 10, 19, 12, 0, 0, 0, time.UTC) for _, tc := range []struct { name string until time.Time @@ -84,7 +84,10 @@ func TestMinorVerifiedUntilLoadValidation(t *testing.T) { {"valid cache", now.Add(5 * time.Minute), true, true}, {"expired cache", now.Add(-time.Minute), true, true}, {"year zero", time.Date(0, 1, 1, 0, 0, 0, 0, time.UTC), true, false}, - {"excessive lifetime", now.Add(24 * time.Hour), true, false}, + {"legacy ten second lifecycle", now.Add(5*time.Minute + 10*time.Second), true, true}, + {"legacy fifteen second lifecycle", now.Add(5*time.Minute + 15*time.Second), true, true}, + {"legacy slow IO and lock lifecycle", now.Add(5*time.Minute + 25*time.Second), true, true}, + {"legacy backward clock", now.Add(24 * time.Hour), true, true}, {"unverified cache", now.Add(time.Minute), false, false}, } { t.Run(tc.name, func(t *testing.T) { @@ -100,6 +103,23 @@ func TestMinorVerifiedUntilLoadValidation(t *testing.T) { if (e == nil) != tc.valid { t.Fatalf("load error = %v, valid = %v", e, tc.valid) } + if tc.name == "valid cache" { + for _, raw := range []string{`42`, `{}`, `"10000-01-01T00:00:00Z"`, `"2026-13-01T00:00:00Z"`} { + var shape map[string]json.RawMessage + if e := json.Unmarshal(mustMarshal(st.Subscriptions[id]), &shape); e != nil { + t.Fatal(e) + } + shape["verified_until"] = json.RawMessage(raw) + var stateShape map[string]json.RawMessage + if e := json.Unmarshal(data, &stateShape); e != nil { + t.Fatal(e) + } + stateShape["subscriptions"] = mustMarshal(map[string]any{id: shape}) + if _, e := parsePersistenceState(mustMarshal(stateShape)); e == nil { + t.Errorf("accepted invalid deadline schema %s", raw) + } + } + } }) } } diff --git a/internal/dotevents/state.go b/internal/dotevents/state.go index e1e805d..d765e9d 100644 --- a/internal/dotevents/state.go +++ b/internal/dotevents/state.go @@ -81,9 +81,10 @@ func parsePersistenceState(data []byte) (*persistenceState, error) { } for id, sub := range st.Subscriptions { if sub != nil && !sub.VerifiedUntil.IsZero() { - // Zero is valid for legacy records and invalidated caches. Older - // records stamped UpdatedAt before the bounded verification I/O. - if !sub.CallbackVerified || sub.VerifiedUntil.Year() < 1 || sub.VerifiedUntil.Year() > 9999 || sub.VerifiedUntil.After(sub.UpdatedAt.Add(5*time.Minute+maxHTTPTimeout)) { + // Validate the persisted shape, not elapsed wall time. Legacy + // UpdatedAt predates lock/save/HTTP work, and wall clocks can step + // in either direction. Cache reuse bounds the deadline separately. + if !sub.CallbackVerified || sub.VerifiedUntil.Year() < 1 || sub.VerifiedUntil.Year() > 9999 { return nil, errf("corrupt") } } From 7e9612095b5764b9b393801c733af438a0cebfb4 Mon Sep 17 00:00:00 2001 From: Pyuyi <204892921+VeraPyuyi@users.noreply.github.com> Date: Mon, 19 Oct 2026 14:18:53 +0800 Subject: [PATCH 19/47] test(dotbridge): verify complete proof and artifact receipts Address scoped review test gaps by comparing the complete persisted per-mode proof after completion replay and validating HTTP result identity, terminal state, text and artifact URL/label. Focused race checks passed in 6.71 seconds; retained dot-proof-review-assertions.log. No live Dot execution claimed. --- internal/dotbridge/task_mode_test.go | 10 +++++++++- internal/dotbridge/task_proof_test.go | 4 ++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/internal/dotbridge/task_mode_test.go b/internal/dotbridge/task_mode_test.go index 0fc40ea..20b3fb6 100644 --- a/internal/dotbridge/task_mode_test.go +++ b/internal/dotbridge/task_mode_test.go @@ -223,9 +223,17 @@ func TestTaskModeHTTPIndependentProofs(t *testing.T) { c = f.claim(t, job.JobID) f.complete(t, c, dotqueue.Result{Text: "fixture result", Artifacts: []dotqueue.Artifact{{URL: "https://example.com/result", Label: "fixture artifact"}}}) code, raw = f.request(t, "GET", "/dot-bridge/v1/jobs/"+job.JobID, nil) - if code != 200 || !bytes.Contains(raw, []byte("fixture artifact")) { + var completed dotbridge.JobDTO + if code != 200 || json.Unmarshal(raw, &completed) != nil { t.Fatalf("task result: %d %s", code, raw) } + if completed.Version != 1 || completed.JobID != job.JobID || completed.RequestID != job.RequestID || completed.RequestDigest != job.RequestDigest || completed.SessionID != job.SessionID || completed.Mode != "task" || completed.State != "completed" || completed.Result == nil { + t.Fatalf("completed result identity/state mismatch: %#v", completed) + } + result := completed.Result + if result.Text != "fixture result" || result.ErrorCode != "" || result.Cancelled || len(result.Artifacts) != 1 || result.Artifacts[0].URL != "https://example.com/result" || result.Artifacts[0].Label != "fixture artifact" { + t.Fatalf("completed result payload mismatch: %#v", result) + } // A queued cancellation is terminal; a claimed cancellation remains intent // until the consumer explicitly acknowledges it. for _, claimed := range []bool{false, true} { diff --git a/internal/dotbridge/task_proof_test.go b/internal/dotbridge/task_proof_test.go index e1849d5..1c6d343 100644 --- a/internal/dotbridge/task_proof_test.go +++ b/internal/dotbridge/task_proof_test.go @@ -62,8 +62,8 @@ func testProbeReplay(t *testing.T, mode string) { } s.mu.Lock() defer s.mu.Unlock() - if !s.state.Clients[id].proof(mode).Verified.Equal(past) { - t.Fatal("first verification timestamp replaced") + if got := s.state.Clients[id].proof(mode); got != proof { + t.Fatalf("completion replay changed persisted proof: got %#v, want %#v", got, proof) } } From 581eb55f050578b3ed6c617eb93f5799ebc8a676 Mon Sep 17 00:00:00 2001 From: Pyuyi <204892921+VeraPyuyi@users.noreply.github.com> Date: Mon, 19 Oct 2026 14:21:38 +0800 Subject: [PATCH 20/47] docs(dotbridge): explain independent answer and task probes Replace staging-only task descriptions with the implemented mode-specific nonce probe, one-time proof consumption, bounded expiry, and durable background result workflow. Retain the distinction between authenticated workflow proof and actual account/computer acceptance. --- docs/DOT_BRIDGE.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/DOT_BRIDGE.md b/docs/DOT_BRIDGE.md index 8e36f6f..7377f37 100644 --- a/docs/DOT_BRIDGE.md +++ b/docs/DOT_BRIDGE.md @@ -6,11 +6,11 @@ Run `dotcontrol bridge pair` to create a separate client credential and queue. T Connect the private Dot plugin, refresh its metadata, and subscribe to `dot.task.available` using the returned queue ID. Events parameters are flat: `name`, `arguments`, `delivery`, and optional `cursor`/`ttlMs`. The SDK handles the Events RPC methods and protocol metadata checks. Only a successful modern `2026-07-28` `server/discover` response advertises `capabilities.events`. -Run `dotcontrol bridge probe ` explicitly to enqueue a nonce challenge. Use `dotcontrol bridge probe --mode answer` for the default text readiness probe. `--mode task` is reserved for a future task verification probe and remains unavailable until an authorized task proof path is implemented; there is no manual readiness setter. Through MCP, claim that job with `dot_task_claim`, then complete it with the exact requested nonce. The service checks the claim token, queue authorization, current verified callback subscription generation, and completed answer state. Expected nonce text is hashed in bridge policy; the challenge prompt remains in the encrypted queue. +Run `dotcontrol bridge probe ` explicitly to enqueue a nonce challenge. Use `dotcontrol bridge probe --mode answer` for the default text readiness probe. Use `--mode task` for an independent background capability probe: it asks the configured Dot workflow to write a nonce to a temporary file in its own workspace, read it back, remove the file, and return the nonce. Through MCP, claim the job with `dot_task_claim`, perform its instructions, and complete it using `dot_task_complete`. The service checks the claim token, queue authorization, current verified callback subscription generation, matching mode, and completed state. Expected nonce text is hashed in bridge policy; the challenge prompt remains in the encrypted queue. There is no manual readiness setter. Each proof is consumed once; replaying a completion cannot extend its expiry. The paired Bearer credential accesses `/dot-bridge/v1/status`, `GET /dot-bridge/v1/jobs` (metadata-only job list for the paired client, newest first; full `result` bodies require `GET /dot-bridge/v1/jobs/{id}`), `/jobs/{id}`, `/requests/{request_id}`, and `/jobs/{id}/cancel` on the numeric loopback listener. CLI bridge client requests use a 10s timeout, `Proxy: nil`, and refuse redirects. No client-supplied identity selects a queue. Status exposes no prompt or credential. A successful text probe enables text readiness for at most ten minutes, bounded by subscription expiry. Pause, revoke, disable, renewal, and unsubscribe invalidate or suppress readiness. Runtime restart requires a fresh subscription and probe; live claim bindings are intentionally not restored. -Regular submission requires current readiness. Task execution remains unavailable until task verification is proven: `capabilities.tasks` is false and task submissions return `task_verification_required`. A text ping establishes no evidence about cloud task execution. An authorized task verification policy and work test are required before enabling that capability. +Regular submission requires current proof for the requested mode. An answer probe enables only `capabilities.text`; a task probe enables only `capabilities.tasks`. Each capability expires within ten minutes and no later than its bound subscription. Starting a probe for one mode preserves a still-valid proof for the other mode. Background submission returns HTTP 202 with the durable job identity immediately; query that same job for progress and results. An answer ping does not enable background work. A returned task nonce proves a response through the authenticated configured workflow; it cannot attest cryptographically that a particular model or computer performed the instructions. Real ChatGPT/Dot account acceptance remains a separate verification step. All MCP callers sharing the authenticated installation hop act as the same configured consumer. Queue IDs and worker IDs provide no authentication by themselves. A worker ID supplies retry identity and is combined with the trusted consumer identity. This proves a round trip through the configured source, not cryptographic model or personal identity. Fixture tests provide protocol evidence only, with no real account acceptance. From 07db630ea8f540db2b077d947f9e6bed4654e9ec Mon Sep 17 00:00:00 2001 From: Pyuyi <204892921+VeraPyuyi@users.noreply.github.com> Date: Mon, 19 Oct 2026 14:22:10 +0800 Subject: [PATCH 21/47] docs(dotbridge): clarify pairing file command lifecycle Explain that Magpie imports into the OS keyring while DotControl task commands still read the protected pairing file. Avoid recommending deletion followed by commands that depend on that file; distinguish file removal from credential revocation. --- docs/DOT_BRIDGE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/DOT_BRIDGE.md b/docs/DOT_BRIDGE.md index 7377f37..e8a7b4f 100644 --- a/docs/DOT_BRIDGE.md +++ b/docs/DOT_BRIDGE.md @@ -2,7 +2,7 @@ The bridge is optional. On a fresh install, run `dotcontrol bridge enable` once to set `dot_bridge_enabled` in the installed local configuration, restart the existing runtime, then run `dotcontrol bridge enable` again to turn the bridge on in the running service. The default installation retains 13 MCP tools. With the bridge available, MCP adds four task tools and three Events methods. A missing or unavailable vault key, or a bridge open failure, leaves the regular runtime and its 13 tools available; bridge admin operations report `bridge_unavailable`. -Run `dotcontrol bridge pair` to create a separate client credential and queue. The command prints only the pairing file path and public IDs (`client_id`, `queue_id`); it never prints the bearer token. Import the file locally into the client, then remove it when import is complete. Pairing locally authorizes the runtime's configured Dot consumer to act on that queue. Pairing grants no desktop, history, or terminal permission. +Run `dotcontrol bridge pair` to create a separate client credential and queue. The command prints only the pairing file path and public IDs (`client_id`, `queue_id`); it never prints the bearer token. Import the file locally into Magpie, which saves its credential in the OS keyring. DotControl's `bridge jobs`, `get`, and `cancel` commands still read the protected pairing file; retain it if you use those commands. You may remove the file after import when you will manage tasks exclusively through Magpie. Revoke the client credential explicitly when access is no longer needed. Pairing locally authorizes the runtime's configured Dot consumer to act on that queue. Pairing grants no desktop, history, or terminal permission. Connect the private Dot plugin, refresh its metadata, and subscribe to `dot.task.available` using the returned queue ID. Events parameters are flat: `name`, `arguments`, `delivery`, and optional `cursor`/`ttlMs`. The SDK handles the Events RPC methods and protocol metadata checks. Only a successful modern `2026-07-28` `server/discover` response advertises `capabilities.events`. From c690372c54a5124680ffddfd6a83274ba7b0b14d Mon Sep 17 00:00:00 2001 From: Pyuyi <204892921+VeraPyuyi@users.noreply.github.com> Date: Mon, 19 Oct 2026 14:28:22 +0800 Subject: [PATCH 22/47] feat(dotbridge): expose scoped job metadata lists and align HTTP budgets --- internal/dotbridge/adapters.go | 4 + internal/dotbridge/api.go | 29 ++- internal/dotbridge/list_api.go | 40 +++++ internal/dotbridge/list_api_test.go | 270 ++++++++++++++++++++++++++++ 4 files changed, 340 insertions(+), 3 deletions(-) create mode 100644 internal/dotbridge/list_api.go create mode 100644 internal/dotbridge/list_api_test.go diff --git a/internal/dotbridge/adapters.go b/internal/dotbridge/adapters.go index d7158ad..36d4187 100644 --- a/internal/dotbridge/adapters.go +++ b/internal/dotbridge/adapters.go @@ -13,6 +13,7 @@ type QueueAPI interface { SetEnabled(enabled bool) error Submit(clientID string, sub dotqueue.Submission) (dotqueue.Job, error) Get(clientID, jobID string) (dotqueue.Job, error) + List(clientID string) ([]dotqueue.Job, error) ByRequest(clientID, requestID string) (dotqueue.Job, error) Claim(queueID, jobID, workerID string) (dotqueue.Claim, error) Progress(jobID, token, state, summary string) (dotqueue.Job, error) @@ -47,6 +48,9 @@ func (q queueAdapter) Submit(clientID string, sub dotqueue.Submission) (dotqueue func (q queueAdapter) Get(clientID, jobID string) (dotqueue.Job, error) { return q.Store.Get(clientID, jobID) } +func (q queueAdapter) List(clientID string) ([]dotqueue.Job, error) { + return q.Store.List(clientID) +} func (q queueAdapter) ByRequest(clientID, requestID string) (dotqueue.Job, error) { return q.Store.ByRequest(clientID, requestID) } diff --git a/internal/dotbridge/api.go b/internal/dotbridge/api.go index dc5a5ff..fa6ade3 100644 --- a/internal/dotbridge/api.go +++ b/internal/dotbridge/api.go @@ -1,6 +1,7 @@ package dotbridge import ( + "bytes" "encoding/json" "errors" "io" @@ -10,7 +11,9 @@ import ( "strings" ) -const maxBody = 128 << 10 +// Allow 256KiB of raw content escaped at six bytes per byte, plus envelope +// overhead. The queue independently enforces the raw content limit. +const maxBody = 2 << 20 func (s *Service) Handler() http.Handler { mux := http.NewServeMux() @@ -65,6 +68,10 @@ func (s *Service) handleStatus(w http.ResponseWriter, r *http.Request) { } func (s *Service) handleJobs(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodGet { + s.handleListJobs(w, r) + return + } if r.Method != http.MethodPost { writeError(w, http.StatusMethodNotAllowed, "method_rejected", "") return @@ -77,7 +84,15 @@ func (s *Service) handleJobs(w http.ResponseWriter, r *http.Request) { } job, code, err := s.submit(client, req) if err != nil { - writeError(w, http.StatusBadRequest, mapError(err), job.JobID) + errCode := mapError(err) + status := http.StatusBadRequest + switch errCode { + case "conflict": + status = http.StatusConflict + case "queue_full": + status = http.StatusTooManyRequests + } + writeError(w, status, errCode, job.JobID) return } writeJSON(w, code, job) @@ -140,7 +155,11 @@ func (s *Service) handleRequestByID(w http.ResponseWriter, r *http.Request) { } func decodeSubmit(body io.Reader) (SubmitRequest, error) { - dec := json.NewDecoder(io.LimitReader(body, maxBody)) + data, err := io.ReadAll(io.LimitReader(body, maxBody+1)) + if err != nil || len(data) > maxBody { + return SubmitRequest{}, errors.New("invalid_request") + } + dec := json.NewDecoder(bytes.NewReader(data)) dec.DisallowUnknownFields() var req SubmitRequest if dec.Decode(&req) != nil { @@ -168,6 +187,10 @@ func mapError(err error) string { } msg := err.Error() switch { + case strings.Contains(msg, "task_probe_required"): + return "task_probe_required" + case strings.Contains(msg, "queue_full"): + return "queue_full" case strings.Contains(msg, "probe_required"): return "probe_required" case strings.Contains(msg, "bridge_disabled"): diff --git a/internal/dotbridge/list_api.go b/internal/dotbridge/list_api.go new file mode 100644 index 0000000..91ca73b --- /dev/null +++ b/internal/dotbridge/list_api.go @@ -0,0 +1,40 @@ +package dotbridge + +import "net/http" + +// JobListDTO contains retained job metadata in newest-first order. Results are +// omitted to bound the list response; GET /dot-bridge/v1/jobs/{id} returns the +// full result. The queue retains at most 100 jobs across all clients. +type JobListDTO struct { + Version int `json:"version"` + Jobs []JobDTO `json:"jobs"` +} + +func (s *Service) listJobs(client ClientPolicy) (JobListDTO, error) { + if err := s.clientGate(client.ClientID); err != nil { + return JobListDTO{}, err + } + jobs, err := s.queue.List(client.ClientID) + // Recheck even when queue I/O fails so an intervening gate change wins. + if gateErr := s.clientGate(client.ClientID); gateErr != nil { + return JobListDTO{}, gateErr + } + if err != nil { + return JobListDTO{}, err + } + out := JobListDTO{Version: APIVersion, Jobs: make([]JobDTO, 0, len(jobs))} + for _, job := range jobs { + job.Result = nil + out.Jobs = append(out.Jobs, toJobDTO(job)) + } + return out, nil +} + +func (s *Service) handleListJobs(w http.ResponseWriter, r *http.Request) { + jobs, err := s.listJobs(clientFrom(r.Context())) + if err != nil { + writeError(w, http.StatusBadRequest, mapError(err), "") + return + } + writeJSON(w, http.StatusOK, jobs) +} diff --git a/internal/dotbridge/list_api_test.go b/internal/dotbridge/list_api_test.go new file mode 100644 index 0000000..aa67a4f --- /dev/null +++ b/internal/dotbridge/list_api_test.go @@ -0,0 +1,270 @@ +package dotbridge + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + + "github.com/dotcontrol/dotcontrol/internal/dotqueue" +) + +type listInterceptQueue struct { + QueueAPI + calls *atomic.Int32 + after func() +} + +func (q listInterceptQueue) List(id string) ([]dotqueue.Job, error) { + q.calls.Add(1) + jobs, err := q.QueueAPI.List(id) + if q.after != nil { + q.after() + } + return jobs, err +} +func listHTTPRequest(t *testing.T, api *httptest.Server, method, path, token string, body []byte) (int, []byte) { + t.Helper() + req, err := http.NewRequest(method, api.URL+path, bytes.NewReader(body)) + if err != nil { + t.Fatal(err) + } + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + res, err := api.Client().Do(req) + if err != nil { + t.Fatal(err) + } + defer res.Body.Close() + raw, err := io.ReadAll(res.Body) + if err != nil { + t.Fatal(err) + } + return res.StatusCode, raw +} +func TestListAPIOwnerMetadata(t *testing.T) { + s, token, id := openTestBridge(t) + pair, _, err := s.Pair("client.b") + if err != nil { + t.Fatal(err) + } + api := httptest.NewServer(s.Handler()) + defer api.Close() + code, raw := listHTTPRequest(t, api, "GET", "/dot-bridge/v1/jobs", token, nil) + if code != 200 || !bytes.Contains(raw, []byte(`"jobs":[]`)) { + t.Fatalf("empty: %d %s", code, raw) + } + owners := []string{id, pair.ClientID} + tokens := []string{token, pair.Token} + jobs := make([][]dotqueue.Job, 2) + var claimToken string + for n, owner := range owners { + for i := 0; i < 2; i++ { + job, err := s.queue.Submit(owner, dotqueue.Submission{RequestID: fmt.Sprintf("counter.%d", i), SessionID: fmt.Sprintf("session.%d", n), Mode: dotqueue.ModeAnswer, Messages: []dotqueue.Message{{Role: "user", Content: fmt.Sprintf("private-prompt-%d", n)}}}) + if err != nil { + t.Fatal(err) + } + jobs[n] = append(jobs[n], job) + } + claim, err := s.queue.Claim(owner, jobs[n][0].ID, "worker") + if err != nil { + t.Fatal(err) + } + claimToken = claim.Token + if _, err = s.queue.Complete(jobs[n][0].ID, claim.Token, dotqueue.Result{Text: strings.Repeat("<", 262144)}); err != nil { + t.Fatal(err) + } + } + for n := range owners { + code, raw = listHTTPRequest(t, api, "GET", "/dot-bridge/v1/jobs", tokens[n], nil) + var env struct { + Version int `json:"version"` + Jobs []JobDTO `json:"jobs"` + } + if code != 200 || json.Unmarshal(raw, &env) != nil || env.Version != 1 || len(env.Jobs) != 2 { + t.Fatalf("list: %d %s", code, raw) + } + if env.Jobs[0].JobID != jobs[n][1].ID || env.Jobs[1].JobID != jobs[n][0].ID { + t.Fatalf("order: %#v", env.Jobs) + } + for _, forbidden := range []string{"private-prompt", claimToken, tokens[0], tokens[1], jobs[1-n][0].ID, jobs[1-n][1].ID, fmt.Sprintf("session.%d", 1-n), `"result"`, `"messages"`, `"task"`, `"token"`} { + if bytes.Contains(raw, []byte(forbidden)) { + t.Fatalf("list leaked %q", forbidden) + } + } + if len(raw) > 4096 { + t.Fatalf("unbounded result in list: %d", len(raw)) + } + code, raw = listHTTPRequest(t, api, "GET", "/dot-bridge/v1/jobs/"+jobs[n][0].ID, tokens[n], nil) + var full JobDTO + if code != 200 || json.Unmarshal(raw, &full) != nil || full.Result == nil || len(full.Result.Text) != 262144 { + t.Fatalf("full result: status %d", code) + } + } +} +func TestListAPIGates(t *testing.T) { + for _, condition := range []string{"missing_auth", "bad_auth", "revoked", "disabled", "paused", "paused_after", "disabled_after", "revoked_after"} { + t.Run(condition, func(t *testing.T) { + s, token, id := openTestBridge(t) + var paused atomic.Bool + s.paused = paused.Load + var calls atomic.Int32 + alter := func() { + switch strings.TrimSuffix(condition, "_after") { + case "paused": + paused.Store(true) + case "disabled": + s.mu.Lock() + s.state.Enabled = false + s.mu.Unlock() + case "revoked": + if err := s.RevokeClient(id); err != nil { + t.Fatal(err) + } + } + } + var after func() + if strings.HasSuffix(condition, "_after") { + after = alter + } else { + alter() + } + s.queue = listInterceptQueue{QueueAPI: s.queue, calls: &calls, after: after} + api := httptest.NewServer(s.Handler()) + defer api.Close() + if condition == "missing_auth" { + token = "" + } + if condition == "bad_auth" { + token = "wrong" + } + code, raw := listHTTPRequest(t, api, "GET", "/dot-bridge/v1/jobs", token, nil) + if code == 200 || bytes.Contains(raw, []byte(`"jobs"`)) { + t.Fatalf("gate disclosed list: %d %s", code, raw) + } + want := int32(0) + if after != nil { + want = 1 + } + if calls.Load() != want { + t.Fatalf("queue reads %d want %d", calls.Load(), want) + } + }) + } +} +func readyListAPIClient(t *testing.T, s *Service, id string) { + t.Helper() + probe, err := s.StartProbe(id) + if err != nil { + t.Fatal(err) + } + claim, err := s.ClaimTask(id, probe.JobID, "worker.ready") + if err != nil { + t.Fatal(err) + } + answer := strings.TrimPrefix(claim.Submission.Messages[0].Content, "Reply with exactly this nonce: ") + if _, err = s.CompleteTask(probe.JobID, claim.Token, dotqueue.Result{Text: answer}); err != nil { + t.Fatal(err) + } +} +func TestSubmitEncodedBudget(t *testing.T) { + for _, character := range []string{"a", "<", "\x01"} { + t.Run(fmt.Sprintf("byte_%02x", character[0]), func(t *testing.T) { + // Construct wire bytes independently of the production JSON encoder. + escaped := character + if character == "<" { + escaped = `\u003c` + } + if character == "\x01" { + escaped = `\u0001` + } + messages := make([]string, 32) + for i := range messages { + messages[i] = `{"role":"user","content":"` + strings.Repeat(escaped, 8192) + `"}` + } + wire := `{"request_id":"budget.1","mode":"answer","session_id":"chat","messages":[` + strings.Join(messages, ",") + `]}` + req, err := decodeSubmit(strings.NewReader(wire)) + if err != nil { + t.Fatal(err) + } + s, token, id := openTestBridge(t) + readyListAPIClient(t, s, id) + api := httptest.NewServer(s.Handler()) + defer api.Close() + code, raw := listHTTPRequest(t, api, "POST", "/dot-bridge/v1/jobs", token, []byte(wire)) + if code != http.StatusAccepted { + t.Fatalf("full-budget HTTP submission: %d %s", code, raw) + } + sub := dotqueue.Submission{RequestID: req.RequestID, Mode: req.Mode, SessionID: req.SessionID} + for _, m := range req.Messages { + sub.Messages = append(sub.Messages, dotqueue.Message{Role: m.Role, Content: m.Content}) + } + if _, err = s.queue.Submit(id, sub); err != nil { + t.Fatalf("raw queue validation: %v", err) + } + sub.RequestID = "over.raw" + sub.Messages[0].Content += "a" + if _, err = s.queue.Submit(id, sub); err == nil { + t.Fatal("queue accepted raw content over 256KiB") + } + }) + } +} + +type submitCountingQueue struct { + QueueAPI + calls *atomic.Int32 + err error +} + +func (q submitCountingQueue) Submit(id string, sub dotqueue.Submission) (dotqueue.Job, error) { + q.calls.Add(1) + return dotqueue.Job{}, q.err +} +func TestSubmitWireRejectionsBeforeQueue(t *testing.T) { + s, token, _ := openTestBridge(t) + var calls atomic.Int32 + s.queue = submitCountingQueue{QueueAPI: s.queue, calls: &calls} + api := httptest.NewServer(s.Handler()) + defer api.Close() + const budget = 2 << 20 + valid := `{"request_id":"counter.1","mode":"answer","messages":[{"role":"user","content":"hi"}]}` + bodies := []string{valid + strings.Repeat(" ", budget+1-len(valid)), valid + ` {}`, `{"unknown":true}`} + for _, body := range bodies { + code, raw := listHTTPRequest(t, api, "POST", "/dot-bridge/v1/jobs", token, []byte(body)) + if code != 400 || !bytes.Contains(raw, []byte("invalid_request")) { + t.Fatalf("rejection: %d %s", code, raw) + } + } + if calls.Load() != 0 { + t.Fatalf("rejected requests wrote queue %d times", calls.Load()) + } + if _, err := decodeSubmit(strings.NewReader(valid + strings.Repeat(" ", budget-len(valid)))); err != nil { + t.Fatalf("exact budget: %v", err) + } +} +func TestSubmitTypedErrors(t *testing.T) { + s, token, id := openTestBridge(t) + readyListAPIClient(t, s, id) + original := s.queue + api := httptest.NewServer(s.Handler()) + defer api.Close() + for _, tc := range []struct { + code string + status int + }{{"conflict", 409}, {"queue_full", 429}, {"task_probe_required", 400}, {"bridge_paused", 400}} { + var calls atomic.Int32 + s.queue = submitCountingQueue{QueueAPI: original, calls: &calls, err: errors.New(tc.code)} + code, raw := listHTTPRequest(t, api, "POST", "/dot-bridge/v1/jobs", token, []byte(`{"request_id":"typed.1","mode":"answer","messages":[{"role":"user","content":"hello"}]}`)) + if code != tc.status || !bytes.Contains(raw, []byte(`"code":"`+tc.code+`"`)) || bytes.Contains(raw, []byte("accepted")) { + t.Fatalf("%s: %d %s", tc.code, code, raw) + } + } +} From 563d238b617135fd2793e874e55434ed3981370d Mon Sep 17 00:00:00 2001 From: Pyuyi <204892921+VeraPyuyi@users.noreply.github.com> Date: Mon, 19 Oct 2026 14:31:27 +0800 Subject: [PATCH 23/47] docs: link the optional Dot task bridge and acceptance limits --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index d5f1d2a..1442bba 100644 --- a/README.md +++ b/README.md @@ -38,6 +38,10 @@ 有人使用电脑时，只允许经过执行器明确支持的后台动作，并避开正在使用的目标窗口。前台操作默认要求连续空闲 60 秒；人工接管、锁屏或活动检测失效时停止后续输入。后台能力因应用而异。 +## 可选 Dot 任务桥接（开发分支） + +[Dot 任务桥接](docs/DOT_BRIDGE.md)允许本机客户端提交持久任务，由连接 ChatGPT 的 Dot 通过 MCP Events 收到通知，再领取并回写结果。回答与后台任务分别验证；Magpie 集成使用独立配对凭据。该功能尚未发布，也尚未完成真实 ChatGPT／Dot 往返验收。它与下面的目标端 Codex 委派使用不同的执行路径。 + ## 目标端 Codex 委派 实现路线：源 Dot → 目标 DotControl → 目标 Codex → 官方 Computer Use／本机 SSH。 From e437da698106acf4f2e16eca4d6c3becea2c5566 Mon Sep 17 00:00:00 2001 From: Pyuyi <204892921+VeraPyuyi@users.noreply.github.com> Date: Mon, 19 Oct 2026 14:38:55 +0800 Subject: [PATCH 24/47] test(dotbridge): verify exact budget, write-after-pause and list error privacy --- internal/dotbridge/http_budget_test.go | 304 +++++++++++++++++++++++++ internal/dotbridge/list_api_test.go | 21 -- 2 files changed, 304 insertions(+), 21 deletions(-) create mode 100644 internal/dotbridge/http_budget_test.go diff --git a/internal/dotbridge/http_budget_test.go b/internal/dotbridge/http_budget_test.go new file mode 100644 index 0000000..a240f80 --- /dev/null +++ b/internal/dotbridge/http_budget_test.go @@ -0,0 +1,304 @@ +package dotbridge + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync" + "sync/atomic" + "testing" + + "github.com/dotcontrol/dotcontrol/internal/dotqueue" +) + +const exactBudget = 2 << 20 + +func listHTTPRequest(t *testing.T, api *httptest.Server, method, path, token string, body []byte) (int, []byte) { + t.Helper() + req, err := http.NewRequest(method, api.URL+path, bytes.NewReader(body)) + if err != nil { + t.Fatal(err) + } + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + res, err := api.Client().Do(req) + if err != nil { + t.Fatal(err) + } + defer res.Body.Close() + raw, err := io.ReadAll(res.Body) + if err != nil { + t.Fatal(err) + } + return res.StatusCode, raw +} + +// budgetPayload builds a valid submit object padded with legal JSON whitespace +// to an exact encoded size. Content bytes are constructed directly, never +// serialized by the production encoder. +func budgetPayload(t *testing.T, requestID, content string, size int) []byte { + t.Helper() + head := `{"request_id":"` + requestID + `","mode":"answer","session_id":"chat","messages":[{"role":"user","content":"` + content + `"}]}` + body := []byte(head) + if len(body) >= size { + t.Fatalf("payload %d exceeds budget %d", len(body), size) + } + body = append(body, []byte(strings.Repeat(" ", size-len(body)))...) + if len(body) != size { + t.Fatalf("built %d want %d", len(body), size) + } + if _, err := decodeSubmit(bytes.NewReader(body)); err != nil { + t.Fatalf("constructed body invalid: %v", err) + } + return body +} + +func countOwnerJobs(t *testing.T, s *Service, clientID string) int { + t.Helper() + jobs, err := s.queue.List(clientID) + if err != nil { + t.Fatal(err) + } + return len(jobs) +} + +func readyHTTPClient(t *testing.T, s *Service, id string) { + t.Helper() + probe, err := s.StartProbe(id) + if err != nil { + t.Fatal(err) + } + claim, err := s.ClaimTask(id, probe.JobID, "worker.ready") + if err != nil { + t.Fatal(err) + } + answer := strings.TrimPrefix(claim.Submission.Messages[0].Content, "Reply with exactly this nonce: ") + if _, err = s.CompleteTask(probe.JobID, claim.Token, dotqueue.Result{Text: answer}); err != nil { + t.Fatal(err) + } +} + +// persistedContent returns total stored raw prompt bytes through a real claim, +// the only public path exposing submission content. +func persistedContent(t *testing.T, s *Service, queueID, jobID, worker string) int { + t.Helper() + claim, err := s.queue.Claim(queueID, jobID, worker) + if err != nil { + t.Fatalf("claim persisted job: %v", err) + } + total := 0 + for _, m := range claim.Submission.Messages { + total += len(m.Content) + } + return total +} + +func submitReceipt(t *testing.T, api *httptest.Server, token string, body []byte) (int, []byte, JobDTO) { + t.Helper() + code, raw := listHTTPRequest(t, api, "POST", "/dot-bridge/v1/jobs", token, body) + var dto JobDTO + if code == http.StatusAccepted && json.Unmarshal(raw, &dto) != nil { + t.Fatalf("receipt: %s", raw) + } + return code, raw, dto +} + +// TestExactBudgetThroughRealHandler submits an exact 2 MiB body and the next +// byte through the real handler against a queue-reachable ready client. +func TestExactBudgetThroughRealHandler(t *testing.T) { + s, token, id := openTestBridge(t) + readyHTTPClient(t, s, id) + api := httptest.NewServer(s.Handler()) + defer api.Close() + code, raw, accepted := submitReceipt(t, api, token, []byte(`{"request_id":"control.1","mode":"answer","session_id":"chat","messages":[{"role":"user","content":"hello"}]}`)) + if code != http.StatusAccepted || !accepted.Accepted { + t.Fatalf("control: %d %s", code, raw) + } + if countOwnerJobs(t, s, id) != 2 { + t.Fatalf("control job not persisted: %d", countOwnerJobs(t, s, id)) + } + exact := budgetPayload(t, "budget.exact", strings.Repeat("a", 256*1024), exactBudget) + code, raw, accepted = submitReceipt(t, api, token, exact) + if code != http.StatusAccepted || !accepted.Accepted { + t.Fatalf("exact 2MiB: %d %s", code, raw) + } + if countOwnerJobs(t, s, id) != 3 { + t.Fatalf("exact budget job not persisted: %d", countOwnerJobs(t, s, id)) + } + if got := persistedContent(t, s, id, accepted.JobID, "worker.audit.exact"); got != 256*1024 { + t.Fatalf("persisted raw content %d", got) + } + over := append(append([]byte(nil), exact...), ' ') + code, raw, _ = submitReceipt(t, api, token, over) + if code != http.StatusBadRequest || !bytes.Contains(raw, []byte(`"code":"invalid_request"`)) { + t.Fatalf("oversize: %d %s", code, raw) + } + if countOwnerJobs(t, s, id) != 3 { + t.Fatalf("oversize wrote a queue job: %d", countOwnerJobs(t, s, id)) + } +} + +// TestWorstEscapingBudgetThroughRealHandler proves the full 256 KiB raw content +// budget survives the encoder's six-byte escapes. +func TestWorstEscapingBudgetThroughRealHandler(t *testing.T) { + for _, tc := range []struct{ escaped, raw string }{{`\u003c`, "<"}, {`\u0001`, "\x01"}} { + t.Run(fmt.Sprintf("escape_%s", tc.escaped), func(t *testing.T) { + s, token, id := openTestBridge(t) + readyHTTPClient(t, s, id) + api := httptest.NewServer(s.Handler()) + defer api.Close() + // 32 messages of fully escaped bytes totalling exactly 256 KiB raw content. + var msgs []string + for i := 0; i < 32; i++ { + msgs = append(msgs, `{"role":"user","content":"`+strings.Repeat(tc.escaped, 8192)+`"}`) + } + wire := []byte(`{"request_id":"budget.escape","mode":"answer","session_id":"chat","messages":[` + strings.Join(msgs, ",") + `]}`) + if len(wire) > exactBudget { + t.Fatalf("constructed wire %d exceeds budget", len(wire)) + } + code, raw, accepted := submitReceipt(t, api, token, wire) + if code != http.StatusAccepted || !accepted.Accepted { + t.Fatalf("worst escaping: %d %s", code, raw) + } + if got := persistedContent(t, s, id, accepted.JobID, "worker.audit.escape"); got != 256*1024 { + t.Fatalf("raw content %d", got) + } + }) + } +} + +// writeAfterAccessQueue persists the real submission, then flips service access +// state. The write is real; the state change happens only after it. +type writeAfterAccessQueue struct { + QueueAPI + mutate func() + once sync.Once +} + +func (q *writeAfterAccessQueue) Submit(clientID string, sub dotqueue.Submission) (dotqueue.Job, error) { + job, err := q.QueueAPI.Submit(clientID, sub) + if err == nil { + q.once.Do(q.mutate) + } + return job, err +} + +// TestWriteAfterPauseStaysConservative proves a durable write followed by a +// pause is never reported accepted and the original job survives unchanged. +func TestWriteAfterPauseStaysConservative(t *testing.T) { + s, token, id := openTestBridge(t) + readyHTTPClient(t, s, id) + api := httptest.NewServer(s.Handler()) + defer api.Close() + var paused atomic.Bool + s.paused = paused.Load + s.queue = &writeAfterAccessQueue{QueueAPI: s.queue, mutate: func() { paused.Store(true) }} + code, raw, _ := submitReceipt(t, api, token, []byte(`{"request_id":"write.pause","mode":"answer","session_id":"chat","messages":[{"role":"user","content":"hello"}]}`)) + if code == http.StatusAccepted || bytes.Contains(raw, []byte("accepted")) { + t.Fatalf("accepted after write: %d %s", code, raw) + } + var env ErrorDTO + if json.Unmarshal(raw, &env) != nil || env.Error.Code != "bridge_paused" { + t.Fatalf("envelope: %s", raw) + } + if len(env.Error.JobID) != 0 { + t.Fatalf("job id disclosed on rejection: %s", raw) + } + jobs, err := s.queue.List(id) + if err != nil { + t.Fatalf("list: %v", err) + } + same := 0 + written := dotqueue.Job{} + for _, j := range jobs { + if j.RequestID == "write.pause" { + same++ + written = j + } + } + if same != 1 { + t.Fatalf("same-ID jobs %d", same) + } + // The write happened before the pause: the job is persisted and still queued. + if written.State != dotqueue.StateQueued || written.Result != nil { + t.Fatalf("original job not persisted as written: %#v", written) + } + paused.Store(false) + if code, raw, resumed := submitReceipt(t, api, token, []byte(`{"request_id":"write.pause","mode":"answer","session_id":"chat","messages":[{"role":"user","content":"hello"}]}`)); code != http.StatusAccepted { + t.Fatalf("same ID after resume: %d %s", code, raw) + } else if resumed.JobID != written.ID { + t.Fatalf("same ID resolved a different job: %q != %q", resumed.JobID, written.ID) + } + if countOwnerJobs(t, s, id) != len(jobs) { + t.Fatalf("job count changed by retry") + } + if got := persistedContent(t, s, id, written.ID, "worker.audit.write"); got != len("hello") { + t.Fatalf("original job altered: %d bytes", got) + } +} + +// TestListErrorPrivacyAndGatePrecedence covers failed list I/O: the gate wins, +// no queue counts, prompts or secrets leak, unknown errors stay opaque. +func TestListErrorPrivacyAndGatePrecedence(t *testing.T) { + const secret = "vault-secret-material" + for _, condition := range []string{"gate_after", "gate_before"} { + t.Run(condition, func(t *testing.T) { + s, token, id := openTestBridge(t) + if _, err := s.queue.Submit(id, dotqueue.Submission{RequestID: "leak.probe", Mode: dotqueue.ModeAnswer, Messages: []dotqueue.Message{{Role: "user", Content: "private prompt"}}}); err != nil { + t.Fatal(err) + } + var paused atomic.Bool + s.paused = paused.Load + s.queue = &listErrorQueue{QueueAPI: s.queue, secret: secret, mutate: func() { paused.Store(true) }} + if condition == "gate_before" { + paused.Store(true) + } + api := httptest.NewServer(s.Handler()) + defer api.Close() + code, raw := listHTTPRequest(t, api, "GET", "/dot-bridge/v1/jobs", token, nil) + var env ErrorDTO + if code != http.StatusBadRequest || json.Unmarshal(raw, &env) != nil || env.Error.Code != "bridge_paused" { + t.Fatalf("gate precedence: %d %s", code, raw) + } + for _, forbidden := range []string{secret, "private prompt", "messages", "count", "remaining", `"jobs"`} { + if bytes.Contains(raw, []byte(forbidden)) { + t.Fatalf("leaked %q: %s", forbidden, raw) + } + } + }) + } + t.Run("unknown_error", func(t *testing.T) { + s, token, _ := openTestBridge(t) + s.queue = &listErrorQueue{QueueAPI: s.queue, secret: secret} + api := httptest.NewServer(s.Handler()) + defer api.Close() + code, raw := listHTTPRequest(t, api, "GET", "/dot-bridge/v1/jobs", token, nil) + var env ErrorDTO + if code != http.StatusBadRequest || json.Unmarshal(raw, &env) != nil || env.Error.Code != "unknown" || bytes.Contains(raw, []byte(secret)) { + t.Fatalf("unknown error disclosed: %s", raw) + } + }) +} + +type listErrorQueue struct { + QueueAPI + secret string + mutate func() +} + +func (q *listErrorQueue) List(clientID string) ([]dotqueue.Job, error) { + jobs, err := q.QueueAPI.List(clientID) + if q.mutate != nil { + q.mutate() + } + if err != nil { + return jobs, err + } + return nil, errors.New("store failure: " + q.secret) +} diff --git a/internal/dotbridge/list_api_test.go b/internal/dotbridge/list_api_test.go index aa67a4f..356b631 100644 --- a/internal/dotbridge/list_api_test.go +++ b/internal/dotbridge/list_api_test.go @@ -5,7 +5,6 @@ import ( "encoding/json" "errors" "fmt" - "io" "net/http" "net/http/httptest" "strings" @@ -29,26 +28,6 @@ func (q listInterceptQueue) List(id string) ([]dotqueue.Job, error) { } return jobs, err } -func listHTTPRequest(t *testing.T, api *httptest.Server, method, path, token string, body []byte) (int, []byte) { - t.Helper() - req, err := http.NewRequest(method, api.URL+path, bytes.NewReader(body)) - if err != nil { - t.Fatal(err) - } - if token != "" { - req.Header.Set("Authorization", "Bearer "+token) - } - res, err := api.Client().Do(req) - if err != nil { - t.Fatal(err) - } - defer res.Body.Close() - raw, err := io.ReadAll(res.Body) - if err != nil { - t.Fatal(err) - } - return res.StatusCode, raw -} func TestListAPIOwnerMetadata(t *testing.T) { s, token, id := openTestBridge(t) pair, _, err := s.Pair("client.b") From 1b1bfdfe0e288df2aee2f6be0c374c6b5249b967 Mon Sep 17 00:00:00 2001 From: Pyuyi <204892921+VeraPyuyi@users.noreply.github.com> Date: Mon, 19 Oct 2026 14:55:12 +0800 Subject: [PATCH 25/47] test(dotqueue): race two child processes for the last job slot Close the capacity coverage gap recorded in the dot-queue capacity review: no test exercised two distinct new requests competing for the single free slot of a store already holding 99 jobs. Two real child OS processes submit distinct request IDs against a store retaining maxRetainedJobs-1 jobs and are released together by an inherited pipe byte after both publish a ready file, so the winner is decided by the queue lock rather than by a timed sleep. The test asserts exactly one success and exactly one queue_full, a retained count of 100 after reopen with an unchanged CapacityStatus, the winner present and idempotent at capacity, the loser absent, and no plaintext prompt, request, client or job id in the encrypted state file. Fault injection through `go test -overlay` (no repository file modified) confirmed the test fails when the capacity guard is removed, made off-by-one, made unreachable, moved ahead of the idempotency lookup, or when the index, order, ByRequest lookup or returned public job is corrupted. Focused and package verification on go1.27.0 windows/amd64 with the MinGW gcc from C:\msys64\mingw64\bin on PATH: - go vet ./internal/dotqueue/ exit 0 - go test ./internal/dotqueue/ -run ^TestCapacityLastSlotCrossProcess$ -count=1 -v PASS 0.52s - go test ./internal/dotqueue/ -run ^TestCapacityLastSlotCrossProcess$ -count=3 -v PASS 3/3 - go test ./internal/dotqueue/ -run ^TestCapacityLastSlotCrossProcess$ -race -count=1 -v PASS 6.72s - go test ./internal/dotqueue/ -run ^TestCapacityLastSlotCrossProcess$ -race -count=5 ok 41.031s - go test ./internal/dotqueue/ -count=1 ok 6.440s - go test ./internal/dotqueue/ -race -count=1 ok 47.376s Test-only change; no production file and no other test file modified. --- internal/dotqueue/capacity_last_slot_test.go | 276 +++++++++++++++++++ 1 file changed, 276 insertions(+) create mode 100644 internal/dotqueue/capacity_last_slot_test.go diff --git a/internal/dotqueue/capacity_last_slot_test.go b/internal/dotqueue/capacity_last_slot_test.go new file mode 100644 index 0000000..fcf775c --- /dev/null +++ b/internal/dotqueue/capacity_last_slot_test.go @@ -0,0 +1,276 @@ +package dotqueue + +import ( + "bytes" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" +) + +// Two independent child OS processes compete for the single remaining slot of a +// store that already retains maxRetainedJobs-1 jobs, each submitting its own +// distinct request ID. +// +// Synchronization is file/pipe based, not a timed sleep: every child first +// publishes a ready file, then blocks reading one byte from its inherited stdin +// pipe. The parent waits for both ready files and only then writes the release +// byte, so both children are provably parked at the same program point when the +// contention starts. Which one wins the queue lock is then decided by the lock +// itself, never by scheduling luck. +// +// Child processes use only temporary fixture state and a fixed, non-secret test key. +func TestCapacityLastSlotCrossProcess(t *testing.T) { + store, dir := openTestStore(t) + fillStoreJobs(t, store, 0, maxRetainedJobs-1, "client") + if c, err := store.Capacity(); err != nil || c.Retained != maxRetainedJobs-1 || c.Remaining != 1 { + t.Fatalf("pre-race capacity: %v %#v", err, c) + } + + handshake := t.TempDir() + type racer struct { + index int + requestID string + outcome string + jobID string + } + racers := make([]racer, 2) + for i := range racers { + racers[i] = racer{index: i, requestID: fmt.Sprintf("req.last.%d", i)} + } + + cmds := make([]*exec.Cmd, 2) + logs := make([]*bytes.Buffer, 2) + release := make([]*os.File, 2) + ready := make([]string, 2) + result := make([]string, 2) + + // Phase 1: spawn both children and wait until both are parked on the pipe. + for i := range racers { + ready[i] = filepath.Join(handshake, fmt.Sprintf("ready-%d", i)) + result[i] = filepath.Join(handshake, fmt.Sprintf("result-%d", i)) + r, w, err := os.Pipe() + if err != nil { + t.Fatal(err) + } + release[i] = w + logs[i] = &bytes.Buffer{} + cmd := exec.Command(os.Args[0], "-test.run=^TestCapacityLastSlotProcessHelper$", "-test.v") + cmd.Env = append(os.Environ(), + "DOTQUEUE_LAST_SLOT_TEST_DIR="+dir, + "DOTQUEUE_LAST_SLOT_TEST_REQUEST="+racers[i].requestID, + "DOTQUEUE_LAST_SLOT_TEST_READY="+ready[i], + "DOTQUEUE_LAST_SLOT_TEST_RESULT="+result[i], + "DOTQUEUE_LAST_SLOT_TEST_RELEASE=stdin", + ) + cmd.Stdin = r + cmd.Stdout = logs[i] + cmd.Stderr = logs[i] + if err := cmd.Start(); err != nil { + r.Close() + w.Close() + t.Fatal(err) + } + r.Close() + cmds[i] = cmd + } + for i := range racers { + if err := waitForFile(ready[i], 30*time.Second); err != nil { + t.Fatalf("racer %d never became ready: %v %s", i, err, logs[i]) + } + } + + // Phase 2: release both at once and let the queue lock decide the winner. + for i := range racers { + if _, err := release[i].Write([]byte{1}); err != nil { + t.Fatal(err) + } + if err := release[i].Close(); err != nil { + t.Fatal(err) + } + } + for i := range racers { + if err := cmds[i].Wait(); err != nil { + t.Fatalf("racer %d failed: %v %s", i, err, logs[i]) + } + } + + for i := range racers { + data, err := os.ReadFile(result[i]) + if err != nil { + t.Fatalf("racer %d result missing: %v %s", i, err, logs[i]) + } + fields := strings.Fields(string(data)) + if len(fields) == 0 { + t.Fatalf("racer %d result empty: %s", i, logs[i]) + } + racers[i].outcome = fields[0] + if len(fields) > 1 { + racers[i].jobID = fields[1] + } + } + + // Phase 3: exactly one success and exactly one queue_full. + success, rejected := 0, 0 + winner, loser := -1, -1 + for i := range racers { + switch racers[i].outcome { + case "ok": + success++ + winner = i + if racers[i].jobID == "" { + t.Fatalf("racer %d reported ok without a job id", i) + } + case "queue_full": + rejected++ + loser = i + if racers[i].jobID != "" { + t.Fatalf("racer %d reported queue_full with a job id %q", i, racers[i].jobID) + } + default: + t.Fatalf("racer %d unexpected outcome %q %s", i, racers[i].outcome, logs[i]) + } + } + if success != 1 || rejected != 1 { + t.Fatalf("last slot outcomes: success=%d queue_full=%d", success, rejected) + } + + // Phase 4: the winning write is durable and caps the global retained count. + cap, err := store.Capacity() + if err != nil || cap.Retained != maxRetainedJobs || cap.Remaining != 0 || cap.Limit != maxRetainedJobs { + t.Fatalf("capacity after race: %v %#v", err, cap) + } + reopened, err := Open(dir, testKey) + if err != nil { + t.Fatal(err) + } + c2, err := reopened.Capacity() + if err != nil || c2 != cap { + t.Fatalf("capacity changed on reopen: before=%#v after=%#v err=%v", cap, c2, err) + } + + // Phase 5: winner present and stable after reopen, loser absent. + job, err := reopened.ByRequest("client.child", racers[winner].requestID) + if err != nil { + t.Fatalf("winner lookup: %v", err) + } + if job.ID != racers[winner].jobID || job.RequestID != racers[winner].requestID || + job.ClientID != "client.child" || job.QueueID != "client.child" || job.State != StateQueued { + t.Fatalf("winner job after reopen: %#v", job) + } + if loserJob := racers[loser].jobID; loserJob != "" { + t.Fatalf("loser returned a job id %q", loserJob) + } + if _, err := reopened.ByRequest("client.child", racers[loser].requestID); errorCode(err) != "notfound" { + t.Fatalf("loser present after reopen: %v", err) + } + + // Phase 6: at full capacity the winner stays idempotent and the loser stays refused. + if res := runLastSlotChild(t, dir, racers[winner].requestID); res != "ok "+racers[winner].jobID { + t.Fatalf("winner retry at capacity: %q", res) + } + if res := runLastSlotChild(t, dir, racers[loser].requestID); res != "queue_full" { + t.Fatalf("loser retry at capacity: %q", res) + } + again, err := reopened.ByRequest("client.child", racers[winner].requestID) + if err != nil || again.ID != racers[winner].jobID || again.RequestDigest != job.RequestDigest { + t.Fatalf("winner changed after idempotent retry: %v %#v", err, again) + } + if _, err := reopened.ByRequest("client.child", racers[loser].requestID); errorCode(err) != "notfound" { + t.Fatalf("loser present after refused retry: %v", err) + } + if c3, err := reopened.Capacity(); err != nil || c3 != cap { + t.Fatalf("capacity changed by refused retries: %v %#v", err, c3) + } + + // Phase 7: the retained state stays encrypted on disk. + data, err := os.ReadFile(filepath.Join(dir, stateFileName)) + if err != nil { + t.Fatal(err) + } + for _, secret := range []string{"hello dot", racers[winner].requestID, racers[loser].requestID, + "req.cap.", "client.child", "sess.1", racers[winner].jobID} { + if strings.Contains(string(data), secret) { + t.Fatalf("plaintext %q found in state file", secret) + } + } +} + +// runLastSlotChild submits requestID through a fresh child process with no +// release synchronization, for post-race assertions at capacity. +func runLastSlotChild(t *testing.T, dir, requestID string) string { + t.Helper() + cmd := exec.Command(os.Args[0], "-test.run=^TestCapacityLastSlotProcessHelper$", "-test.v") + cmd.Env = append(os.Environ(), + "DOTQUEUE_LAST_SLOT_TEST_DIR="+dir, + "DOTQUEUE_LAST_SLOT_TEST_REQUEST="+requestID, + ) + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("child failed: %v %s", err, out) + } + body := strings.TrimSpace(string(out)) + for _, line := range strings.Split(body, "\n") { + if v, ok := strings.CutPrefix(strings.TrimSpace(line), "RESULT="); ok { + return strings.TrimSpace(v) + } + } + t.Fatalf("child result missing: %s", body) + return "" +} + +func waitForFile(path string, timeout time.Duration) error { + deadline := time.Now().Add(timeout) + for { + if _, err := os.Stat(path); err == nil { + return nil + } + if time.Now().After(deadline) { + return fmt.Errorf("timeout waiting for %s", path) + } + time.Sleep(2 * time.Millisecond) + } +} + +// TestCapacityLastSlotProcessHelper is the child-process entry point. Without +// fixture environment it is a no-op, so it never interferes with normal runs. +func TestCapacityLastSlotProcessHelper(t *testing.T) { + dir := os.Getenv("DOTQUEUE_LAST_SLOT_TEST_DIR") + if dir == "" { + return + } + requestID := os.Getenv("DOTQUEUE_LAST_SLOT_TEST_REQUEST") + if requestID == "" { + t.Fatal("missing request id") + } + if os.Getenv("DOTQUEUE_LAST_SLOT_TEST_RELEASE") == "stdin" { + if err := os.WriteFile(os.Getenv("DOTQUEUE_LAST_SLOT_TEST_READY"), []byte("ready\n"), 0600); err != nil { + t.Fatal(err) + } + buf := make([]byte, 1) + if _, err := os.Stdin.Read(buf); err != nil { + t.Fatal(err) + } + } + store, err := Open(dir, testKey) + if err != nil { + t.Fatal(err) + } + job, err := store.Submit("client.child", sampleSubmission(requestID)) + line := "queue_full" + if err == nil { + line = "ok " + job.ID + } else if errorCode(err) != "queue_full" { + line = "error " + errorCode(err) + } + if p := os.Getenv("DOTQUEUE_LAST_SLOT_TEST_RESULT"); p != "" { + if err := os.WriteFile(p, []byte(line+"\n"), 0600); err != nil { + t.Fatal(err) + } + return + } + fmt.Println("RESULT=" + line) +} From d437577770a09c7a2cf76e60bfbd0fc251c79efc Mon Sep 17 00:00:00 2001 From: Pyuyi <204892921+VeraPyuyi@users.noreply.github.com> Date: Mon, 19 Oct 2026 15:22:27 +0800 Subject: [PATCH 26/47] docs: Magpie dot pairing identity implementation report --- .../dot-pairing-identity-report.md | 67 +++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 externalresearch/dot-pairing-identity-report.md diff --git a/externalresearch/dot-pairing-identity-report.md b/externalresearch/dot-pairing-identity-report.md new file mode 100644 index 0000000..8e250c9 --- /dev/null +++ b/externalresearch/dot-pairing-identity-report.md @@ -0,0 +1,67 @@ +# Magpie Dot pairing identity fix (I1/I2) + +Repository: `dot-automatic-routing` (Magpie whole-branch). Scope: `internal/dotbridge`, `internal/gateway/dot.go`, `internal/cli/dot_cli.go`, `internal/provider/dot.go` (eligibility via `CachedReady` only). GUI files were not edited; GUI worker should wire `dotbridge.PairingIdentity(cfg)` the same way as CLI. + +## Problem + +- **I1:** Process-global readiness cache ignored pairing changes made in another process or after off/pair/re-pair. +- **I2:** Gateway/CLI journal scopes used caller key + agent (CLI: address + credential ref) without a stable pairing generation, so the same client request ID after re-pair collided with the old queue namespace. + +## Design + +### Public pairing identity + +- `Config` gains `pairing_generation` (random 16-byte hex). **Never** derived from token or token hash. +- `SavePairing` / credential replacement / address or credential ref change / off / clear always mint a **new** generation (`saveConfig(..., replace true)` for pairing). +- `SaveConfig` mints generation only when enabled/address/credential change; toggling `answer_wait_seconds` alone does not rotate generation. +- Legacy configs with credential but empty generation: `LoadConfig` migrates once under `dot-bridge.json.lock` (fail-closed if lock unavailable). + +`PairingIdentity(cfg)` = `pair-` + SHA256(JSON `[canonicalLoopbackURL, credentialRef, pairingGeneration]`). Canonical address uses numeric loopback URL from `parseLoopbackAddress` (e.g. `http://127.0.0.1:9`). + +### Readiness cache (I1) + +- Cache stores `identity` + status + timestamp + generation. +- `Connection.Status` publishes only when `c.pairingIdentity != ""` and `LoadConfig()` still matches that identity and enabled. +- `CachedReady` reloads config and requires `PairingIdentity(cfg) == readyCache.identity` plus existing proof checks. +- `invalidateReady()` on pairing identity or enabled change in `saveConfig`; `SavePairing`/`ClearPairing` use locked paths. +- `NewConnection` without `PairedConnection` does **not** write production readiness. +- `PairedConnection` binds `pairingIdentity` and rejects token load when config identity diverges mid-flight. + +### Namespaces (I2) + +- Gateway `dotScope` = `pairingIdentity + ":" + hash(keyID, agent)` (request context carries identity from `prepareDot`). +- CLI `dotScope` = `cli:` + `PairingIdentity(cfg)`. +- `serveDot` refuses enqueue when request identity ≠ `conn.PairingIdentity()` ≠ fresh `LoadConfig()`. +- Old journal records remain under old scope; new pairing does not auto-replay or erase them. + +## GUI integration contract (for parallel worker) + +```go +// After dotbridge.LoadConfig(): +scope := "gui:" + dotbridge.PairingIdentity(cfg) // or hash if GUI prefers fixed width +// Replace internal sha256(address+"\x00"+credential) scope helper with PairingIdentity(cfg). +// Empty identity => unmigrated/invalid; do not enqueue. +``` + +`Connection.PairingIdentity()` returns the identity bound at `PairedConnection()` time. + +## Tests added + +- `internal/dotbridge/pairing_identity_test.go`: external atomic config write, inflight status, unbound client, off/re-enable, distinct generation same ref. +- `internal/gateway/dot_pairing_identity_test.go`: stable client request ID after `SavePairing` gets new scope/session/request IDs; old journal intact. + +## Verification (reviewed head, local) + +```text +go test ./internal/dotbridge ./internal/provider ./internal/gateway ./internal/cli -run Dot -count=1 +``` + +All passed. Not run: full `go test ./...`, `-race`, live cfg/vault, GUI. + +## Files touched (implementation) + +- `internal/dotbridge/config.go`, `client.go` +- `internal/dotbridge/pairing_identity_test.go`, `review_test.go` +- `internal/gateway/dot.go`, `dot_pairing_identity_test.go`, `dot_testutil_test.go` +- `internal/cli/dot_cli.go`, `dot_review_test.go` +- `internal/provider/dot_test.go` From 2642ef7ebf562f6933b5065cbb61094a86a4b1ad Mon Sep 17 00:00:00 2001 From: Pyuyi <204892921+VeraPyuyi@users.noreply.github.com> Date: Mon, 19 Oct 2026 15:23:28 +0800 Subject: [PATCH 27/47] Expose bridge capacity and job summaries for operators --- cmd/dotcontrol/dot_bridge.go | 16 ++- cmd/dotcontrol/dot_bridge_response_test.go | 76 +++++++++++ .../dot-status-progress-report.md | 72 +++++++++++ internal/dotbridge/adapters.go | 4 + internal/dotbridge/api.go | 8 +- internal/dotbridge/job_summary_test.go | 88 +++++++++++++ internal/dotbridge/service.go | 42 +++++- internal/dotbridge/status_capacity_test.go | 120 ++++++++++++++++++ internal/dotbridge/types.go | 10 ++ 9 files changed, 428 insertions(+), 8 deletions(-) create mode 100644 cmd/dotcontrol/dot_bridge_response_test.go create mode 100644 externalresearch/dot-status-progress-report.md create mode 100644 internal/dotbridge/job_summary_test.go create mode 100644 internal/dotbridge/status_capacity_test.go diff --git a/cmd/dotcontrol/dot_bridge.go b/cmd/dotcontrol/dot_bridge.go index 925657f..e2a0511 100644 --- a/cmd/dotcontrol/dot_bridge.go +++ b/cmd/dotcontrol/dot_bridge.go @@ -187,7 +187,10 @@ func bridgeClientRequest(ctx context.Context, listen, token, method, path string return err } defer res.Body.Close() - data, _ := io.ReadAll(io.LimitReader(res.Body, 1<<20)) + data, err := readBridgeResponseBody(res.Body) + if err != nil { + return err + } fmt.Println(string(data)) if res.StatusCode >= 400 { return errors.New("bridge_request_failed") @@ -203,6 +206,17 @@ func loadPairingToken(dir, clientID string) (string, error) { return file.Token, nil } +func readBridgeResponseBody(r io.Reader) ([]byte, error) { + data, err := io.ReadAll(io.LimitReader(r, dotbridge.MaxWireResponseBytes+1)) + if err != nil { + return nil, err + } + if len(data) > dotbridge.MaxWireResponseBytes { + return nil, errors.New("bridge_response_too_large") + } + return data, nil +} + func validateBridgeListen(listen string) error { host, port, err := net.SplitHostPort(listen) if err != nil { diff --git a/cmd/dotcontrol/dot_bridge_response_test.go b/cmd/dotcontrol/dot_bridge_response_test.go new file mode 100644 index 0000000..cf089ed --- /dev/null +++ b/cmd/dotcontrol/dot_bridge_response_test.go @@ -0,0 +1,76 @@ +package main + +import ( + "bytes" + "context" + "errors" + "io" + "net/http" + "net/http/httptest" + "os" + "strings" + "testing" + + "github.com/dotcontrol/dotcontrol/internal/dotbridge" +) + +type errReader struct{ err error } + +func (e errReader) Read([]byte) (int, error) { return 0, e.err } + +func TestBridgeClientResponseEscapedResultComplete(t *testing.T) { + body := strings.Repeat("x", 1573333) + data, err := readBridgeResponseBody(strings.NewReader(body)) + if err != nil || len(data) != len(body) { + t.Fatalf("escaped result: err=%v len=%d", err, len(data)) + } +} + +func TestReadBridgeResponseBodyExactAndOverflowBudget(t *testing.T) { + exact := bytes.Repeat([]byte{'a'}, dotbridge.MaxWireResponseBytes) + data, err := readBridgeResponseBody(bytes.NewReader(exact)) + if err != nil || len(data) != dotbridge.MaxWireResponseBytes { + t.Fatalf("exact 2MiB: err=%v len=%d", err, len(data)) + } + over := append(exact, 'b') + _, err = readBridgeResponseBody(bytes.NewReader(over)) + if err == nil || err.Error() != "bridge_response_too_large" { + t.Fatalf("overflow: %v", err) + } +} + +func TestReadBridgeResponseBodyReadError(t *testing.T) { + _, err := readBridgeResponseBody(errReader{errors.New("read_failed")}) + if err == nil { + t.Fatal("expected read error") + } +} + +func TestBridgeClientResponseReadFailurePrintsNothing(t *testing.T) { + srv := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hj, ok := w.(http.Hijacker) + if !ok { + t.Fatal("hijack unavailable") + } + conn, _, err := hj.Hijack() + if err != nil { + t.Fatal(err) + } + _, _ = conn.Write([]byte("HTTP/1.1 200 OK\r\nContent-Length: 100\r\n\r\n")) + _, _ = conn.Write([]byte("partial")) + _ = conn.Close() + })) + srv.Start() + defer srv.Close() + old := os.Stdout + r, w, _ := os.Pipe() + os.Stdout = w + err := bridgeClientRequest(context.Background(), strings.TrimPrefix(srv.URL, "http://"), "fixture-token", http.MethodGet, "/dot-bridge/v1/jobs/job.1", nil) + _ = w.Close() + os.Stdout = old + var printed strings.Builder + _, _ = io.Copy(&printed, r) + if err == nil || printed.String() != "" { + t.Fatalf("read failure: err=%v out=%q", err, printed.String()) + } +} diff --git a/externalresearch/dot-status-progress-report.md b/externalresearch/dot-status-progress-report.md new file mode 100644 index 0000000..b2aab5b --- /dev/null +++ b/externalresearch/dot-status-progress-report.md @@ -0,0 +1,72 @@ +# Dot status/progress/capacity crosswire report + +Date: 2026-10-19. Scope: whole-branch review items I3, I4, and I5 only. + +## Summary + +DotControl now exposes enabled-gated aggregate queue capacity through admin and paired status, carries durable progress summaries through the version-1 job projection and local GET/request/list surfaces, and fixes the operator CLI response budget so valid escaped results are not silently truncated. + +Magpie adds matching DTO fields for `Job.summary` and `Status.capacity` so controller/GUI consumers can render additive status without changing identity-worker ownership. + +## DotControl changes + +- `internal/dotbridge/adapters.go` + - Added `QueueAPI.Capacity()` and queue adapter delegation. +- `internal/dotbridge/types.go` + - Added `CapacityDTO` and `StatusDTO.capacity`. + - Added `JobDTO.summary`. +- `internal/dotbridge/service.go` + - Added `bridgeCapacity()` with enabled gate before and after queue I/O. + - `AdminStatus()` and `statusDTO()` now include aggregate capacity when enabled. + - `toJobDTO()` now projects `Job.Summary`. +- `internal/dotbridge/api.go` + - Exported `MaxWireResponseBytes` for shared CLI/client budget alignment. +- `cmd/dotcontrol/dot_bridge.go` + - Added `readBridgeResponseBody()` with max+1 overflow detection and read-error propagation. + - Removed the old 1 MiB silent truncation path. +- Tests + - `internal/dotbridge/status_capacity_test.go` + - `internal/dotbridge/job_summary_test.go` + - `cmd/dotcontrol/dot_bridge_response_test.go` + +## Magpie changes + +- `internal/dotbridge/types.go` + - Added `Capacity` and `Status.capacity`. + - Added `Job.summary`. +- `internal/dotbridge/types_test.go` + - Added JSON field coverage for the new DTO fields. + +## Behavior notes + +- Capacity is aggregate-only: `limit`, `retained`, `remaining`. +- Capacity is omitted when the bridge is disabled, preventing disabled-state count leakage. +- Probe jobs and terminal jobs count toward retained capacity. +- `queue_full` remains the stable human-facing rejection reason at the storage boundary. +- Job summaries are bounded by the existing queue store limit (2048 UTF-8 bytes) and are retained on GET, request lookup, and list metadata. +- Server raw result budget (256 KiB) and encoded wire budget (2 MiB) were not changed. + +## Verification + +DotControl: + +```powershell +& "C:\Program Files\Go\bin\go.exe" test ./internal/dotbridge/ -run "TestJobSummary|TestAdminAndPaired|TestCapacityFull" -count=1 -v +& "C:\Program Files\Go\bin\go.exe" test ./cmd/dotcontrol/ -run "TestBridgeClient|TestReadBridge" -count=1 -v +& "C:\Program Files\Go\bin\go.exe" test ./internal/dotbridge/ -count=1 +``` + +Magpie: + +```powershell +& "C:\Program Files\Go\bin\go.exe" test ./internal/dotbridge/ -run TestJobAndStatusSummaryCapacityFields -count=1 -v +``` + +Observed: all scoped tests PASS. + +## Out of scope + +- I1 readiness cache identity binding +- I2 gateway journal namespace pairing identity +- GUI rendering/controller wiring beyond additive DTO fields +- Archive/eviction semantics for retained jobs diff --git a/internal/dotbridge/adapters.go b/internal/dotbridge/adapters.go index 36d4187..6afed8c 100644 --- a/internal/dotbridge/adapters.go +++ b/internal/dotbridge/adapters.go @@ -21,6 +21,7 @@ type QueueAPI interface { Cancel(clientID, jobID string) (dotqueue.Job, error) Recover() error Pending(queueID string) ([]dotqueue.Notice, error) + Capacity() (dotqueue.CapacityStatus, error) } // EventsAPI isolates Task3 integration from dotevents manager details. @@ -70,6 +71,9 @@ func (q queueAdapter) Recover() error { return q.Store.Recover() } func (q queueAdapter) Pending(queueID string) ([]dotqueue.Notice, error) { return q.Store.Pending(queueID) } +func (q queueAdapter) Capacity() (dotqueue.CapacityStatus, error) { + return q.Store.Capacity() +} func (e eventsAdapter) SetEnabled(enabled bool) error { return e.Manager.SetEnabled(enabled) } func (e eventsAdapter) Subscribe(ctx context.Context, principal string, params dotevents.Params) (dotevents.Subscription, error) { diff --git a/internal/dotbridge/api.go b/internal/dotbridge/api.go index fa6ade3..fd26241 100644 --- a/internal/dotbridge/api.go +++ b/internal/dotbridge/api.go @@ -11,9 +11,11 @@ import ( "strings" ) -// Allow 256KiB of raw content escaped at six bytes per byte, plus envelope -// overhead. The queue independently enforces the raw content limit. -const maxBody = 2 << 20 +// MaxWireResponseBytes allows 256KiB of raw content escaped at six bytes per +// byte, plus envelope overhead. The queue independently enforces the raw limit. +const MaxWireResponseBytes = 2 << 20 + +const maxBody = MaxWireResponseBytes func (s *Service) Handler() http.Handler { mux := http.NewServeMux() diff --git a/internal/dotbridge/job_summary_test.go b/internal/dotbridge/job_summary_test.go new file mode 100644 index 0000000..9f95e94 --- /dev/null +++ b/internal/dotbridge/job_summary_test.go @@ -0,0 +1,88 @@ +package dotbridge + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "github.com/dotcontrol/dotcontrol/internal/dotqueue" +) + +func TestJobSummarySurfacesOnGetListAndRequest(t *testing.T) { + s, token, clientID := openTestBridge(t) + api := httptest.NewServer(s.Handler()) + defer api.Close() + + job, err := s.queue.Submit(clientID, dotqueue.Submission{ + RequestID: "summary.req", + SessionID: "session.summary", + Mode: dotqueue.ModeAnswer, + Messages: []dotqueue.Message{{Role: "user", Content: "work"}}, + }) + if err != nil { + t.Fatal(err) + } + claim, err := s.ClaimTask(clientID, job.ID, "worker.summary") + if err != nil { + t.Fatal(err) + } + summary := "waiting for operator confirmation" + if _, err = s.ProgressTask(job.ID, claim.Token, dotqueue.StateWaitingUser, summary); err != nil { + t.Fatal(err) + } + + paths := []string{ + "/dot-bridge/v1/jobs/" + job.ID, + "/dot-bridge/v1/requests/" + job.RequestID, + } + for _, path := range paths { + code, raw := listHTTPRequest(t, api, http.MethodGet, path, token, nil) + var dto JobDTO + if code != http.StatusOK || json.Unmarshal(raw, &dto) != nil || dto.Summary != summary || dto.State != dotqueue.StateWaitingUser { + t.Fatalf("%s: %d %#v %s", path, code, dto, raw) + } + } + + code, raw := listHTTPRequest(t, api, http.MethodGet, "/dot-bridge/v1/jobs", token, nil) + var list JobListDTO + if code != http.StatusOK || json.Unmarshal(raw, &list) != nil || len(list.Jobs) != 1 { + t.Fatalf("list: %d %s", code, raw) + } + if list.Jobs[0].Summary != summary { + t.Fatalf("list summary %#v", list.Jobs[0]) + } +} + +func TestJobSummaryProgressToJobDTO(t *testing.T) { + s, _, clientID := openTestBridge(t) + job, err := s.queue.Submit(clientID, dotqueue.Submission{ + RequestID: "dto.req", + SessionID: "session.dto", + Mode: dotqueue.ModeAnswer, + Messages: []dotqueue.Message{{Role: "user", Content: "x"}}, + }) + if err != nil { + t.Fatal(err) + } + claim, err := s.ClaimTask(clientID, job.ID, "worker.dto") + if err != nil { + t.Fatal(err) + } + want := "running step 2 of 5" + updated, err := s.ProgressTask(job.ID, claim.Token, dotqueue.StateRunning, want) + if err != nil { + t.Fatal(err) + } + dto := toJobDTO(updated) + if dto.Summary != want { + t.Fatalf("toJobDTO summary %q", dto.Summary) + } + s.mu.Lock() + policy := s.state.Clients[clientID] + s.mu.Unlock() + got, err := s.getJob(policy, job.ID) + if err != nil || got.Summary != want { + t.Fatalf("getJob %#v err=%v", got, err) + } +} diff --git a/internal/dotbridge/service.go b/internal/dotbridge/service.go index f4b28a3..4b0ca67 100644 --- a/internal/dotbridge/service.go +++ b/internal/dotbridge/service.go @@ -180,21 +180,50 @@ func (s *Service) Disable() error { return s.saveState() } +func (s *Service) bridgeCapacity() (*CapacityDTO, error) { + s.mu.Lock() + enabled := s.state.Enabled + s.mu.Unlock() + if !enabled { + return nil, nil + } + cap, err := s.queue.Capacity() + if err != nil { + return nil, err + } + s.mu.Lock() + still := s.state.Enabled + s.mu.Unlock() + if !still { + return nil, nil + } + return &CapacityDTO{Limit: cap.Limit, Retained: cap.Retained, Remaining: cap.Remaining}, nil +} + func (s *Service) AdminStatus() (map[string]any, error) { s.mu.Lock() - defer s.mu.Unlock() paired := 0 for _, policy := range s.state.Clients { if !policy.Revoked { paired++ } } - return map[string]any{ + enabled := s.state.Enabled + s.mu.Unlock() + out := map[string]any{ "version": APIVersion, - "enabled": s.state.Enabled, + "enabled": enabled, "paired": paired > 0, "clients": paired, - }, nil + } + cap, err := s.bridgeCapacity() + if err != nil { + return nil, err + } + if cap != nil { + out["capacity"] = cap + } + return out, nil } func (s *Service) StatusForClient(clientID string) (StatusDTO, error) { @@ -212,6 +241,7 @@ func (s *Service) statusDTO(policy ClientPolicy) StatusDTO { // never combine its subscription with a previously captured proof. eventStatus, e := s.events.Status(policy.QueueID, policy.QueueID) gateOK := s.clientGate(policy.ClientID) == nil + capacity, capErr := s.bridgeCapacity() s.mu.Lock() defer s.mu.Unlock() current, exists := s.state.Clients[policy.ClientID] @@ -222,6 +252,9 @@ func (s *Service) statusDTO(policy ClientPolicy) StatusDTO { if active { out.Subscription = SubscriptionDTO{State: "active", ExpiresAt: &sub.Expires} } + if capErr == nil && capacity != nil { + out.Capacity = capacity + } if !gateOK || !s.state.Enabled || !exists || current.Revoked || !active || e != nil || !eventStatus.Subscribed || !eventStatus.CallbackVerified || (s.paused != nil && s.paused()) || (s.revoked != nil && s.revoked()) { return out } @@ -513,6 +546,7 @@ func toJobDTO(job dotqueue.Job) JobDTO { Mode: job.Mode, SessionID: job.SessionID, State: job.State, + Summary: job.Summary, CreatedAt: job.CreatedAt, UpdatedAt: job.UpdatedAt, CancelRequested: job.CancelRequested, diff --git a/internal/dotbridge/status_capacity_test.go b/internal/dotbridge/status_capacity_test.go new file mode 100644 index 0000000..e8f0698 --- /dev/null +++ b/internal/dotbridge/status_capacity_test.go @@ -0,0 +1,120 @@ +package dotbridge + +import ( + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/dotcontrol/dotcontrol/internal/dotqueue" +) + +func TestAdminAndPairedStatusExposeCapacity(t *testing.T) { + s, token, clientID := openTestBridge(t) + api := httptest.NewServer(s.Handler()) + defer api.Close() + + admin, err := s.AdminStatus() + if err != nil { + t.Fatal(err) + } + cap := decodeCapacity(admin["capacity"]) + if cap.Limit != 100 || cap.Retained != 0 || cap.Remaining != 100 { + t.Fatalf("empty admin capacity %#v", cap) + } + + if _, err := s.StartProbe(clientID); err != nil { + t.Fatal(err) + } + admin, err = s.AdminStatus() + if err != nil { + t.Fatal(err) + } + cap = decodeCapacity(admin["capacity"]) + if cap.Retained != 1 || cap.Remaining != 99 { + t.Fatalf("probe retained admin capacity %#v", cap) + } + + code, raw := listHTTPRequest(t, api, http.MethodGet, "/dot-bridge/v1/status", token, nil) + var paired StatusDTO + if code != http.StatusOK || json.Unmarshal(raw, &paired) != nil || paired.Capacity == nil { + t.Fatalf("paired status: %d %s", code, raw) + } + if paired.Capacity.Retained != 1 || paired.Capacity.Remaining != 99 { + t.Fatalf("paired capacity %#v", paired.Capacity) + } +} + +func TestCapacityFullIncludesProbesAndOmitsWhenDisabled(t *testing.T) { + s, token, clientID := openTestBridge(t) + readyListAPIClient(t, s, clientID) + api := httptest.NewServer(s.Handler()) + defer api.Close() + + for i := 0; i < 98; i++ { + _, err := s.queue.Submit(clientID, dotqueue.Submission{ + RequestID: fmt.Sprintf("fill.%d", i), + SessionID: "session.fill", + Mode: dotqueue.ModeAnswer, + Messages: []dotqueue.Message{{Role: "user", Content: "x"}}, + }) + if err != nil { + t.Fatalf("submit %d: %v", i, err) + } + } + if _, err := s.StartProbe(clientID); err != nil { + t.Fatal(err) + } + admin, err := s.AdminStatus() + if err != nil { + t.Fatal(err) + } + cap := decodeCapacity(admin["capacity"]) + if cap.Retained != 100 || cap.Remaining != 0 { + t.Fatalf("full capacity %#v", cap) + } + + if _, err := s.StartProbe(clientID); err == nil || !strings.Contains(err.Error(), "queue_full") { + t.Fatalf("probe overflow: %v", err) + } + _, err = s.queue.Submit(clientID, dotqueue.Submission{ + RequestID: "overflow.req", + SessionID: "session.overflow", + Mode: dotqueue.ModeAnswer, + Messages: []dotqueue.Message{{Role: "user", Content: "y"}}, + }) + if err == nil || !strings.Contains(err.Error(), "queue_full") { + t.Fatalf("queue overflow: %v", err) + } + + if err := s.Disable(); err != nil { + t.Fatal(err) + } + admin, err = s.AdminStatus() + if err != nil { + t.Fatal(err) + } + if admin["capacity"] != nil { + t.Fatalf("disabled admin leaked capacity %#v", admin) + } + code, raw := listHTTPRequest(t, api, http.MethodGet, "/dot-bridge/v1/status", token, nil) + var paired StatusDTO + if code != http.StatusOK || json.Unmarshal(raw, &paired) != nil { + t.Fatalf("status: %d %s", code, raw) + } + if paired.Capacity != nil { + t.Fatalf("disabled paired leaked capacity %#v", paired.Capacity) + } +} + +func decodeCapacity(v any) CapacityDTO { + b, err := json.Marshal(v) + if err != nil { + return CapacityDTO{} + } + var out CapacityDTO + _ = json.Unmarshal(b, &out) + return out +} diff --git a/internal/dotbridge/types.go b/internal/dotbridge/types.go index ac44d07..91bf246 100644 --- a/internal/dotbridge/types.go +++ b/internal/dotbridge/types.go @@ -64,6 +64,14 @@ type PersistedState struct { Clients map[string]ClientPolicy `json:"clients"` } +// CapacityDTO reports global retained job counts for operator visibility. +// It intentionally omits client-identifying fields. +type CapacityDTO struct { + Limit int `json:"limit"` + Retained int `json:"retained"` + Remaining int `json:"remaining"` +} + type StatusDTO struct { Version int `json:"version"` Enabled bool `json:"enabled"` @@ -74,6 +82,7 @@ type StatusDTO struct { ExpiresAt *time.Time `json:"expires_at,omitempty"` Subscription SubscriptionDTO `json:"subscription"` Capabilities CapPolicy `json:"capabilities"` + Capacity *CapacityDTO `json:"capacity,omitempty"` } type SubscriptionDTO struct { @@ -89,6 +98,7 @@ type JobDTO struct { Mode string `json:"mode"` SessionID string `json:"session_id"` State string `json:"state"` + Summary string `json:"summary,omitempty"` CreatedAt time.Time `json:"created_at"` UpdatedAt time.Time `json:"updated_at"` CancelRequested bool `json:"cancel_requested"` From fe5fffa65882e89f524036e16cd3eea4ec49950e Mon Sep 17 00:00:00 2001 From: Pyuyi <204892921+VeraPyuyi@users.noreply.github.com> Date: Mon, 19 Oct 2026 15:33:26 +0800 Subject: [PATCH 28/47] Gate capacity publication on final authorized status snapshot --- cmd/dotcontrol/dot_bridge_response_test.go | 44 +++++- .../dot-status-progress-report.md | 19 +++ internal/dotbridge/service.go | 47 +++--- internal/dotbridge/status_capacity_test.go | 146 ++++++++++++++++++ 4 files changed, 233 insertions(+), 23 deletions(-) diff --git a/cmd/dotcontrol/dot_bridge_response_test.go b/cmd/dotcontrol/dot_bridge_response_test.go index cf089ed..3422a88 100644 --- a/cmd/dotcontrol/dot_bridge_response_test.go +++ b/cmd/dotcontrol/dot_bridge_response_test.go @@ -3,7 +3,9 @@ package main import ( "bytes" "context" + "encoding/json" "errors" + "fmt" "io" "net/http" "net/http/httptest" @@ -19,10 +21,44 @@ type errReader struct{ err error } func (e errReader) Read([]byte) (int, error) { return 0, e.err } func TestBridgeClientResponseEscapedResultComplete(t *testing.T) { - body := strings.Repeat("x", 1573333) - data, err := readBridgeResponseBody(strings.NewReader(body)) - if err != nil || len(data) != len(body) { - t.Fatalf("escaped result: err=%v len=%d", err, len(data)) + want := strings.Repeat("<", 256<<10) + body, err := json.Marshal(dotbridge.JobDTO{Version: 1, JobID: "job.fixture", Result: &dotbridge.ResultDTO{Text: want}}) + if err != nil || len(body) <= 1<<20 { + t.Fatalf("escaped fixture: size=%d err=%v", len(body), err) + } + for _, overflow := range []bool{false, true} { + t.Run(fmt.Sprint(overflow), func(t *testing.T) { + wire := body + if overflow { + wire = append(append([]byte(nil), body...), bytes.Repeat([]byte{' '}, dotbridge.MaxWireResponseBytes+1-len(body))...) + } + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write(wire) })) + defer srv.Close() + // A regular file avoids blocking stdout on a full pipe buffer. + out, err := os.CreateTemp(t.TempDir(), "stdout") + if err != nil { + t.Fatal(err) + } + defer out.Close() + old := os.Stdout + os.Stdout = out + callErr := bridgeClientRequest(context.Background(), strings.TrimPrefix(srv.URL, "http://"), "fixture-token", http.MethodGet, "/dot-bridge/v1/jobs/job.fixture", nil) + os.Stdout = old + data, err := os.ReadFile(out.Name()) + if err != nil { + t.Fatal(err) + } + if overflow { + if callErr == nil || callErr.Error() != "bridge_response_too_large" || len(data) != 0 { + t.Fatalf("overflow: err=%v output=%d", callErr, len(data)) + } + return + } + var got dotbridge.JobDTO + if callErr != nil || json.Unmarshal(data, &got) != nil || got.Result == nil || got.Result.Text != want { + t.Fatalf("incomplete CLI JSON: err=%v bytes=%d", callErr, len(data)) + } + }) } } diff --git a/externalresearch/dot-status-progress-report.md b/externalresearch/dot-status-progress-report.md index b2aab5b..95ee9ef 100644 --- a/externalresearch/dot-status-progress-report.md +++ b/externalresearch/dot-status-progress-report.md @@ -70,3 +70,22 @@ Observed: all scoped tests PASS. - I2 gateway journal namespace pairing identity - GUI rendering/controller wiring beyond additive DTO fields - Archive/eviction semantics for retained jobs + +## Capacity publication review fix — 2026-10-19 + +Addressed Important I1 and the scoped test observations in `dot-status-progress-review.md`. The earlier implementation's enabled checks did not protect the final publication: a revoked client could receive counts and admin enabled/counts could describe different lifecycle snapshots. This was a production defect, not waived as a minor test issue. + +`bridgeCapacity` now returns a candidate plus the service enable epoch, without holding the service mutex during queue I/O. Enable/Disable advance that epoch. AdminStatus builds enabled, client totals and capacity inclusion together under its final lock. Paired status attaches counts only under its final lock when enabled, unpaused, access not revoked, current client still authorized with the same credential identity, and the epoch matches. A disable/re-enable invalidates old counts even if final enabled is true. Client revocation suppresses paired counts and reports paired:false; authenticated admin aggregate visibility survives individual client revocation. Capacity storage errors still propagate for an authorized admin; intervening access changes suppress stale data/error publication. + +RED: `go test ./internal/dotbridge -run TestCapacityPublicationAccessChanges -count=1` failed on paired revoke/pause/access revoke/disable-enable and admin disable/disable-enable/pause/access revoke. GREEN: the same matrix passes after the production change. Deterministic hooks run after real queue I/O and before publication; additional channel-blocked tests execute Disable/RevokeClient in another goroutine before releasing the captured result. Disabled-before tests assert zero Capacity calls. No queue I/O is performed while holding the service mutex. + +CLI fixture now uses json.Marshal of a 256 KiB raw '<' result (encoded body exceeds 1 MiB), serves it over actual HTTP, captures stdout in a regular file and decodes the complete result. Its HTTP overflow variant asserts bridge_response_too_large and zero stdout. Existing exact 2 MiB/+1 helper boundaries and short HTTP body tests remain. + +Verification on the final fix: + +- `go test ./internal/dotbridge ./cmd/dotcontrol -run 'TestCapacity|TestAdminAndPaired|TestBridgeClientResponse|TestReadBridgeResponse' -count=1` PASS. +- `go test -race ./internal/dotbridge ./cmd/dotcontrol -run 'TestCapacity|TestAdminAndPaired|TestBridgeClientResponse|TestReadBridgeResponse' -count=1` PASS (CGO_ENABLED=1, CC=C:\msys64\mingw64\bin\gcc.exe; final run 7.241s / 1.887s). +- `go test ./internal/dotbridge -count=1` PASS (6.620s). +- `git diff --check` PASS. + +No GUI, Magpie identity/client, queue storage/proof, ACL, raw-content or wire-budget production semantics changed. These checks establish the scoped regression fix, not live Dot or whole-branch completion. diff --git a/internal/dotbridge/service.go b/internal/dotbridge/service.go index 4b0ca67..37a595f 100644 --- a/internal/dotbridge/service.go +++ b/internal/dotbridge/service.go @@ -17,6 +17,7 @@ import ( // Service owns bridge lifecycle, local API auth, and delivery worker wiring. type Service struct { mu sync.Mutex + capacityEpoch uint64 // guarded by mu; invalidates reads across enable/disable ctx context.Context cancel context.CancelFunc root string @@ -164,6 +165,7 @@ func (s *Service) Enable() error { return err } s.state.Enabled = true + s.capacityEpoch++ return s.saveState() } @@ -177,31 +179,38 @@ func (s *Service) Disable() error { return err } s.state.Enabled = false + s.capacityEpoch++ return s.saveState() } -func (s *Service) bridgeCapacity() (*CapacityDTO, error) { +// bridgeCapacity captures a candidate only. Callers publish it under mu using +// the same final authorization snapshot as the rest of their response. +func (s *Service) bridgeCapacity() (*CapacityDTO, uint64, error) { s.mu.Lock() - enabled := s.state.Enabled + enabled, epoch := s.capacityAllowedLocked(), s.capacityEpoch s.mu.Unlock() if !enabled { - return nil, nil + return nil, epoch, nil } cap, err := s.queue.Capacity() if err != nil { - return nil, err - } - s.mu.Lock() - still := s.state.Enabled - s.mu.Unlock() - if !still { - return nil, nil + return nil, epoch, err } - return &CapacityDTO{Limit: cap.Limit, Retained: cap.Retained, Remaining: cap.Remaining}, nil + return &CapacityDTO{Limit: cap.Limit, Retained: cap.Retained, Remaining: cap.Remaining}, epoch, nil +} + +func (s *Service) capacityAllowedLocked() bool { + return s.state.Enabled && (s.paused == nil || !s.paused()) && (s.revoked == nil || !s.revoked()) } func (s *Service) AdminStatus() (map[string]any, error) { + cap, epoch, err := s.bridgeCapacity() s.mu.Lock() + defer s.mu.Unlock() + allowed := s.capacityAllowedLocked() && epoch == s.capacityEpoch + if err != nil && allowed { + return nil, err + } paired := 0 for _, policy := range s.state.Clients { if !policy.Revoked { @@ -209,18 +218,13 @@ func (s *Service) AdminStatus() (map[string]any, error) { } } enabled := s.state.Enabled - s.mu.Unlock() out := map[string]any{ "version": APIVersion, "enabled": enabled, "paired": paired > 0, "clients": paired, } - cap, err := s.bridgeCapacity() - if err != nil { - return nil, err - } - if cap != nil { + if allowed && cap != nil { out["capacity"] = cap } return out, nil @@ -241,7 +245,12 @@ func (s *Service) statusDTO(policy ClientPolicy) StatusDTO { // never combine its subscription with a previously captured proof. eventStatus, e := s.events.Status(policy.QueueID, policy.QueueID) gateOK := s.clientGate(policy.ClientID) == nil - capacity, capErr := s.bridgeCapacity() + var capacity *CapacityDTO + var epoch uint64 + var capErr error + if gateOK { + capacity, epoch, capErr = s.bridgeCapacity() + } s.mu.Lock() defer s.mu.Unlock() current, exists := s.state.Clients[policy.ClientID] @@ -252,7 +261,7 @@ func (s *Service) statusDTO(policy ClientPolicy) StatusDTO { if active { out.Subscription = SubscriptionDTO{State: "active", ExpiresAt: &sub.Expires} } - if capErr == nil && capacity != nil { + if gateOK && exists && !current.Revoked && current.TokenHash == policy.TokenHash && s.capacityAllowedLocked() && epoch == s.capacityEpoch && capErr == nil { out.Capacity = capacity } if !gateOK || !s.state.Enabled || !exists || current.Revoked || !active || e != nil || !eventStatus.Subscribed || !eventStatus.CallbackVerified || (s.paused != nil && s.paused()) || (s.revoked != nil && s.revoked()) { diff --git a/internal/dotbridge/status_capacity_test.go b/internal/dotbridge/status_capacity_test.go index e8f0698..09a1cd8 100644 --- a/internal/dotbridge/status_capacity_test.go +++ b/internal/dotbridge/status_capacity_test.go @@ -7,6 +7,7 @@ import ( "net/http/httptest" "strings" "testing" + "time" "github.com/dotcontrol/dotcontrol/internal/dotqueue" ) @@ -109,6 +110,151 @@ func TestCapacityFullIncludesProbesAndOmitsWhenDisabled(t *testing.T) { } } +// The hook runs after the real queue read has released its lock and before +// status publication, so lifecycle calls also prove service.mu is not held. +type capacityInterceptQueue struct { + QueueAPI + after func() + calls int +} + +func (q *capacityInterceptQueue) Capacity() (dotqueue.CapacityStatus, error) { + q.calls++ + out, err := q.QueueAPI.Capacity() + if q.after != nil { + q.after() + } + return out, err +} + +func TestCapacityPublicationAccessChanges(t *testing.T) { + for _, admin := range []bool{false, true} { + for _, change := range []string{"none", "disabled_before", "disable", "disable_enable", "revoke", "pause", "access_revoke"} { + t.Run(fmt.Sprintf("admin=%v/%s", admin, change), func(t *testing.T) { + s, _, id := openTestBridge(t) + var paused, revoked bool + s.paused = func() bool { return paused } + s.revoked = func() bool { return revoked } + q := &capacityInterceptQueue{QueueAPI: s.queue} + s.queue = q + q.after = func() { + switch change { + case "disable", "disable_enable": + if err := s.Disable(); err != nil { + t.Fatal(err) + } + if change == "disable_enable" { + if err := s.Enable(); err != nil { + t.Fatal(err) + } + } + case "revoke": + if err := s.RevokeClient(id); err != nil { + t.Fatal(err) + } + case "pause": + paused = true + case "access_revoke": + revoked = true + } + } + if change == "disabled_before" { + if err := s.Disable(); err != nil { + t.Fatal(err) + } + } + wantCapacity := change == "none" || (admin && change == "revoke") + if admin { + out, err := s.AdminStatus() + if err != nil { + t.Fatal(err) + } + if (out["capacity"] != nil) != wantCapacity { + t.Fatalf("capacity publication: %#v", out) + } + if out["enabled"] != (change != "disable" && change != "disabled_before") { + t.Fatalf("mixed enabled snapshot: %#v", out) + } + } else { + out, err := s.StatusForClient(id) + if err != nil { + t.Fatal(err) + } + if (out.Capacity != nil) != wantCapacity { + t.Fatalf("capacity publication: %#v", out) + } + if change == "revoke" && out.Paired { + t.Fatalf("revoked still paired: %#v", out) + } + } + if change == "disabled_before" && q.calls != 0 { + t.Fatalf("disabled queue reads=%d", q.calls) + } + }) + } + } +} + +func TestCapacityBlockedPublication(t *testing.T) { + for _, admin := range []bool{false, true} { + for _, change := range []string{"disable", "revoke"} { + t.Run(fmt.Sprintf("admin=%v/%s", admin, change), func(t *testing.T) { + s, _, id := openTestBridge(t) + readDone, release := make(chan struct{}), make(chan struct{}) + s.queue = &capacityInterceptQueue{QueueAPI: s.queue, after: func() { close(readDone); <-release }} + type result struct { + status StatusDTO + admin map[string]any + err error + } + done := make(chan result, 1) + go func() { + var r result + if admin { + r.admin, r.err = s.AdminStatus() + } else { + r.status, r.err = s.StatusForClient(id) + } + done <- r + }() + select { + case <-readDone: + case <-time.After(5 * time.Second): + t.Fatal("capacity read blocked by service lock") + } + var err error + if change == "disable" { + err = s.Disable() + } else { + err = s.RevokeClient(id) + } + close(release) + if err != nil { + t.Fatal(err) + } + select { + case r := <-done: + if r.err != nil { + t.Fatal(r.err) + } + if admin { + if change == "disable" && (r.admin["enabled"] != false || r.admin["capacity"] != nil) { + t.Fatalf("disabled admin: %#v", r.admin) + } + if change == "revoke" && (r.admin["paired"] != false || r.admin["capacity"] == nil) { + t.Fatalf("authorized admin aggregate: %#v", r.admin) + } + } else if r.status.Capacity != nil || (change == "revoke" && r.status.Paired) { + t.Fatalf("unauthorized publication: %#v", r.status) + } + case <-time.After(5 * time.Second): + t.Fatal("status did not finish") + } + }) + } + } +} + func decodeCapacity(v any) CapacityDTO { b, err := json.Marshal(v) if err != nil { From 70d2740cba5abc32aa1d4e538ca01a60921b8973 Mon Sep 17 00:00:00 2001 From: Pyuyi <204892921+VeraPyuyi@users.noreply.github.com> Date: Mon, 19 Oct 2026 15:38:21 +0800 Subject: [PATCH 29/47] docs: record pairing identity review corrections and race evidence --- .../dot-pairing-identity-report.md | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/externalresearch/dot-pairing-identity-report.md b/externalresearch/dot-pairing-identity-report.md index 8e250c9..155202e 100644 --- a/externalresearch/dot-pairing-identity-report.md +++ b/externalresearch/dot-pairing-identity-report.md @@ -65,3 +65,31 @@ All passed. Not run: full `go test ./...`, `-race`, live cfg/vault, GUI. - `internal/gateway/dot.go`, `dot_pairing_identity_test.go`, `dot_testutil_test.go` - `internal/cli/dot_cli.go`, `dot_review_test.go` - `internal/provider/dot_test.go` + +## Follow-up: Important review I1/I2/I3 (2026-10-19) + +Read the complete `dot-pairing-identity-review.md` and actual GUI callback implementation at `0ec8af7` before this correction. The original sections above describe ab120e0 and are superseded where they disagree with this section. + +- **I1 compatibility:** Before creating a new Attempt/reservation, gateway preparation derives the exact pre-generation scope from authenticated KeyID + agent and the original wire request/session IDs from the stable request header. If that legacy record exists, the request stops with `legacy_request_manual_recovery_required` or `legacy_request_digest_conflict`, original scope and original wire request ID, and `automatic_redispatch:false`. No GET/POST to the current queue is attempted because the unbound legacy scope cannot prove ownership by that queue. Changed content is compared against the original digest using the original session/request IDs. Legacy records are not edited or erased. `magpie dot archives` / `dotbridge.RequestArchive()` expose all retained records with their original scope and wire ID for explicit manual recovery; archive viewing performs no network operations and never reassigns records to a new pairing. +- **I2 off:** Enabled changes no longer rotate PairingGeneration. Off/list/re-enable/resume keep the same scope and wire ID. A separate public random `readiness_revision` changes on enabled/pairing changes and gates status publication/cache consumption, so an off/on cycle in another process cannot revive cached proof even though durable pairing identity remains stable. Actual SavePairing/replacement still always rotates PairingGeneration. Timeout-only edits preserve both revisions. ClearPairing still revokes credentials and rotates identity. +- **I3 binding:** `BoundPairingSnapshot() (Config, *Connection, error)` captures config and connection from one cross-process config lock, including migration and credential lookup. Disabled snapshots return config, nil, errDisabled. Remote scopes MUST be derived from returned config or connection identity, never an earlier independent load. CLI compares initial identity against captured snapshot and derives scope from connection; gateway uses the same helper and checks prepared identity. Bound `Enqueue` checks current pairing AND scope identity before Reserve; bound `Resume` checks before reading the record/querying. A post-POST switch never clears the Attempt barrier. + +### GUI worker/controller exact integration contract + +Keep initial LoadConfig only for local disabled/off/list behavior. Replace later PairedConnection call with: + +```go +boundCfg, c, err := dotbridge.BoundPairingSnapshot() +if err != nil || dotbridge.PairingIdentity(boundCfg) != dotbridge.PairingIdentity(cfg) { + // Return pairing-changed/unverified before any journal or remote operation. + return +} +scope = "gui:" + c.PairingIdentity() +// All Status / Enqueue / Resume / Cancel calls for this operation use c. +``` + +No GUI file was edited by this worker. `RequestArchive()` returns `[]ArchivedRequest` with `Scope string json:"scope"` plus embedded Snapshot (original request_id/digest/state/session/job/mode/barrier). It is local inspection only; do not feed archive scopes to the current paired connection. Disabled current-scope list remains valid because off preserves PairingGeneration. The exported helper and bound-client scope checks support the GUI worker's concurrent correction. + +### Regression evidence + +Before implementation, real behavioral FAILs: `TestDotLegacyStableHeaderNeverReposts/{same,conflict}` each observed 1 new POST; `TestDotReviewOffPreservesPendingNamespace` reported stranded namespace; `TestDotReviewOffResumeUsesOriginalWireID` made zero GETs and failed request-not-found; `TestDotReviewSwitchBeforeReserveLeavesNoRecord` found a new acceptance_unknown record; `TestDotReviewConnectionRejectsForeignScope` accepted foreign scope. After implementation, focused dotbridge/provider/gateway/CLI Dot suites pass. Additional tests check archive listing, deterministic independent atomic config switch during snapshot construction, and post-POST barrier preservation. Final implementation commit: `a3a339e` in Magpie. Final CGO/MinGW `go test -race ./internal/dotbridge ./internal/provider ./internal/gateway ./internal/cli -run Dot -count=1` passed (dotbridge 6.843s, provider 3.888s, gateway 32.235s, CLI 7.013s). Full `go test ./internal/dotbridge -count=1` passed. `go vet ./internal/dotbridge ./internal/gateway ./internal/cli ./internal/provider`, gofmt, and `git diff --check` passed. First race invocation timed out at 60s; the subsequent complete runs passed with a larger command budget. Full repository suite, live cfg/vault/accounts, and this worker's GUI tests were not run. GUI remains owned by the parallel GUI worker; this commit stages only the eight owned config/client/journal/gateway/CLI implementation and regression files. From 9705d8f25b3b55e4590bec76cf3acdd7cba12940 Mon Sep 17 00:00:00 2001 From: Pyuyi <204892921+VeraPyuyi@users.noreply.github.com> Date: Mon, 19 Oct 2026 15:47:32 +0800 Subject: [PATCH 30/47] fix(dotevents): retry transient Windows state replacement denials Both reported concurrency tests reproduced with unknown-error stacks at saveState replacement. Loaded diagnostics captured MoveFileEx access denied; concurrent directory ACL propagation alone reproduced the same syscall denial with protected source and target ACLs in all three isolation repetitions. Match dotqueue's bounded one-second retry for access-denied and sharing- violation errors under the existing store lock. Add a real Windows handle regression that fails before the fix and passes after reader release, plus missing-source preservation coverage. Retain evidence and limits in the concurrency diagnosis report; remove all temporary instrumentation. Validation: Windows Go 1.27 MinGW race regression x20 PASS (3.356s), entire events package race x30 PASS (159.376s), vet and diff checks PASS. Full-suite race passes events (7.580s) but fails outside scope in cmd/dotcontrol: TestUninstallWaitsForProcessAfterEngineLockRelease context deadline exceeded. --- .../dot-events-concurrency-diagnosis.md | 59 ++++++++++++++++++ internal/dotevents/files_windows.go | 13 +++- internal/dotevents/replace_windows_test.go | 62 +++++++++++++++++++ 3 files changed, 133 insertions(+), 1 deletion(-) create mode 100644 externalresearch/dot-events-concurrency-diagnosis.md create mode 100644 internal/dotevents/replace_windows_test.go diff --git a/externalresearch/dot-events-concurrency-diagnosis.md b/externalresearch/dot-events-concurrency-diagnosis.md new file mode 100644 index 0000000..5850ca9 --- /dev/null +++ b/externalresearch/dot-events-concurrency-diagnosis.md @@ -0,0 +1,59 @@ +# Windows events concurrency diagnosis + +Investigated 2026-10-19 on Windows with Go 1.27 and MinGW (`C:/msys64/mingw64/bin/gcc.exe`), `CGO_ENABLED=1`, `-race`. Assigned baseline was fe5fffa; the shared worktree had advanced to 70d2740 (documentation commit) when investigation began. Existing controller edits to `internal/dotevents/store_test.go` (child output capture and wait-all) and `internal/dotbridge/status_capacity_test.go` were preserved. No subagents or live configurations, vaults, or callback endpoints were used. + +## Cause and proof + +The reproduced failures originate in `store.saveState` when Windows refuses the single `MoveFileEx(REPLACE_EXISTING | WRITE_THROUGH)` attempt with `ERROR_ACCESS_DENIED`. The event-store file lock serializes state operations, but does not prevent temporary Windows handles from denying delete sharing. In particular, another transaction performs `checkAncestors` / `privatePath(dir, true)` before acquiring the lock. That calls `SetNamedSecurityInfo` with inheritable directory ACEs. Windows propagates those to existing children and can overlap the locked writer's replacement. Events previously failed immediately on this temporary denial. Queue already has bounded retries for this exact Windows boundary. + +Evidence is retained in `C:/Users/13678/AppData/Local/Temp/opencode/magpie-context-verification/`: + +- `dot-events-flake-focused-diagnostic-02.log`: 100 repetitions of both originally failing tests under race. Two cross-process failures and two concurrent-subscription failures reproduce. Every captured unknown stack points to `saveState`'s replacement branch (original store.go:168); subscription fails at its first persistence transaction (manager.go:61), before callback verification / generation finalization. Package fails after 149.619s. +- `dot-events-flake-loaded-diagnostic-03.log`: same target tests alongside queue/bridge concurrency/capacity tests, 100 repetitions. Four captured child failures explicitly report `windows_move_file_ex: Access is denied`, followed by `saveState` stacks. Events finishes in 150.408s; other package completion is recorded separately in this log. +- `dot-events-flake-acl-isolation-04.log`: an investigation-only test repeatedly replaces a file while a second goroutine reapplies its parent directory ACL. All three repetitions fail on replacement iterations 1, 1, 3 with access denied. +- `dot-events-flake-acl-protected-isolation-05.log`: repeat with protected current-user ACLs on source and target, matching store privacy setup. All three repetitions fail on iterations 2, 0, 3. This isolates ACL propagation as a sufficient trigger, without HTTP, manager generations, or flock contention. +- `dot-events-flake-regression-red-06.log`: committed regression test holds the target with a real Windows handle allowing read/write sharing but denying delete sharing; closes it after 50ms. Existing single-attempt replacement fails with access denied. The missing-source test also checks that the original target remains intact. +- `dot-events-flake-regression-green-07.log`: both regression tests pass with the retry fix, 20 race repetitions, 3.356s. + +The first attempted run (`dot-events-flake-focused-diagnostic.log`, UTF-16 PowerShell output) did not build because ucrt64 gcc was absent; it is not test evidence. Subsequent runs use the installed mingw64 compiler and UTF-8 logs. + +Temporary instrumentation was installed exclusively by a test init guarded by `DOT_EVENTS_DIAGNOSE=1`; stack locations and Windows stage errors went to test stderr. Public `Error.Error()` remained `dot_events_unknown`. All instrumentation and exploratory isolation tests were removed from the final implementation. + +## Alternatives traced + +- `withLock` checks directory/path privacy, checks and opens the lock file, verifies hardlink count, closes that checked handle, reapplies lock privacy, and acquires a new gofrs/flock handle. The library uses Windows `LockFileEx` and retries lock violation / pending lock acquisition. It does not replace state. Its handle differs from the prechecked handle, unlike queue's checked-handle locking pattern, but captured failures occur after successful acquisition, at replacement. No locking rewrite is justified by this evidence. +- State, marker, and lock-file `protectFile` checks occur after lock acquisition. State reads, protected-file checks and envelope parsing report corrupt on their failures. Directory and lock-file privacy checks occur before locking; the directory update has child propagation effects. Captured unknown stacks exclude failures at those checks themselves. +- `Subscribe` validates parameters and obtains a generation, writes a provisional record, verifies HTTP outside the lock, then finalizes only the matching generation. Concurrent supersession returns stale. Captured subscription unknown failures occur during the initial `saveState`, not in stale-generation logic or HTTP. +- Remaining unknown paths cover invalid keys/paths/parameters, random-token or encryption failure, marshal/size limits, file creation/write/sync/close, marker creation and link validation. Stack diagnostics distinguish these from the observed replacement branch. On Windows `syncDirectory` is a no-op; replacement instrumentation identifies the failing call directly. +- Both original full-suite failures lasted far less than five seconds. No captured `DATA RACE` report supports a Go memory race, and extending the lock timeout would not address the observed syscall denial. + +Microsoft references checked during investigation: + +- [SetNamedSecurityInfoW](https://learn.microsoft.com/en-us/windows/win32/api/aclapi/nf-aclapi-setnamedsecurityinfow), Remarks: changing a DACL automatically propagates inheritable ACEs to existing children. +- [CreateFileW](https://learn.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-createfilew), `dwShareMode`: sharing options last until handle closure; `FILE_SHARE_DELETE` allows delete access, which covers rename operations. + +## Fix + +`internal/dotevents/files_windows.go` now mirrors queue's one-second bounded retry around `MoveFileEx`, restricted to `ERROR_ACCESS_DENIED` and `ERROR_SHARING_VIOLATION`, with a 10ms interval. It retains the same sealed source, destination, replacement flags, existing lock and privacy checks. Other errors return immediately; persistent denials still fail with the existing redacted outward error. No lock timeout or callback fixture timeout changed. + +`internal/dotevents/replace_windows_test.go` exercises a real transient Windows reader and verifies that replacement succeeds, target content changes and source disappears. It also verifies immediate missing-source failure preserves the old target. The controller's child diagnostics remain intact as an existing uncommitted edit; they are excluded from this agent's commit. + +## Validation and limits + +Commands use the explicit executable `C:/Program Files/Go/bin/go.exe`, `CGO_ENABLED=1`, `CC=C:/msys64/mingw64/bin/gcc.exe`, and prepend `C:/msys64/mingw64/bin` to PATH: + +```text +go test -race ./internal/dotevents -run 'TestConcurrentSubscribeAndDeliver|TestCrossProcessLockAndMissingState' -count=100 -timeout=10m +go test -race ./internal/dotevents ./internal/dotqueue ./internal/dotbridge -run 'TestConcurrentSubscribeAndDeliver|TestCrossProcessLockAndMissingState|TestConcurrent|TestCapacity' -count=100 -timeout=10m +go test -race ./internal/dotevents -run '^TestWindowsReplace' -count=20 +go test -race ./internal/dotevents -count=30 -timeout=10m +go test -race ./... -count=1 -timeout=10m +``` + +`dot-events-flake-full-fixed-09.log`: full-suite race run completed. Events passes in 7.580s; queue, bridge and all other packages except cmd/dotcontrol pass. The overall suite is **red** because `TestUninstallWaitsForProcessAfterEngineLockRelease` fails at `install_exit_test.go:89` after 30.11s with `context deadline exceeded`; cmd/dotcontrol finishes failed at 48.670s. This is outside the assigned events scope and was not changed. No DATA RACE report appears. `dot-events-flake-package-fixed-08.log`: all events tests pass for 30 race repetitions in 159.376s, including both previously failing concurrency tests. + +`go vet ./internal/dotevents` passes (exit 0; `dot-events-flake-vet-10.log` is empty). `git diff --check` passes. + +No external `run_checks` executable/tool is exposed in this subagent session; equivalent Go race commands were run directly. + +The ACL experiment proves a sufficient local trigger and manager diagnostics prove the failing syscall; no Windows kernel trace attributes each individual historical denial to a particular ACL propagation handle. Antivirus/scanner handles can produce the same denial, so the fix handles that bounded syscall condition as well. It does not claim to solve arbitrary permanent permission failures or lock-path substitution. The intentionally limited retry can add up to approximately one second per persistently denied replacement. Tests ran on Windows; no Unix execution was performed, and Unix code is unchanged. diff --git a/internal/dotevents/files_windows.go b/internal/dotevents/files_windows.go index e2fb03b..fa35eb9 100644 --- a/internal/dotevents/files_windows.go +++ b/internal/dotevents/files_windows.go @@ -5,6 +5,7 @@ package dotevents import ( "os" "syscall" + "time" "golang.org/x/sys/windows" ) @@ -59,7 +60,17 @@ func replaceFile(from, to string) error { if err != nil { return err } - return windows.MoveFileEx(source, target, windows.MOVEFILE_REPLACE_EXISTING|windows.MOVEFILE_WRITE_THROUGH) + // Directory ACL propagation and other Windows readers can briefly deny + // delete sharing, even while we hold the store lock. Keep the sealed source + // and retry only these transient errors, as the queue store does. + deadline := time.Now().Add(time.Second) + for { + err := windows.MoveFileEx(source, target, windows.MOVEFILE_REPLACE_EXISTING|windows.MOVEFILE_WRITE_THROUGH) + if err == nil || err != windows.ERROR_ACCESS_DENIED && err != windows.ERROR_SHARING_VIOLATION || time.Now().After(deadline) { + return err + } + time.Sleep(10 * time.Millisecond) + } } func syncDirectory(string) error { return nil } diff --git a/internal/dotevents/replace_windows_test.go b/internal/dotevents/replace_windows_test.go new file mode 100644 index 0000000..2eed719 --- /dev/null +++ b/internal/dotevents/replace_windows_test.go @@ -0,0 +1,62 @@ +//go:build windows + +package dotevents + +import ( + "os" + "path/filepath" + "testing" + "time" + + "golang.org/x/sys/windows" +) + +// Windows ACL propagation and scanners can hold a file without delete sharing. +// A transient reader must not cause a locked store update to fail. +func TestWindowsReplaceAfterTransientReader(t *testing.T) { + dir := t.TempDir() + target, source := filepath.Join(dir, "state"), filepath.Join(dir, "next") + if err := os.WriteFile(target, []byte("old"), 0600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(source, []byte("new"), 0600); err != nil { + t.Fatal(err) + } + name, err := windows.UTF16PtrFromString(target) + if err != nil { + t.Fatal(err) + } + handle, err := windows.CreateFile(name, windows.GENERIC_READ, windows.FILE_SHARE_READ|windows.FILE_SHARE_WRITE, nil, windows.OPEN_EXISTING, windows.FILE_ATTRIBUTE_NORMAL, 0) + if err != nil { + t.Fatal(err) + } + released := make(chan struct{}) + go func() { time.Sleep(50 * time.Millisecond); windows.CloseHandle(handle); close(released) }() + defer func() { <-released }() + err = replaceFile(source, target) + if err != nil { + t.Fatal("transient reader prevented replacement:", err) + } + data, err := os.ReadFile(target) + if err != nil || string(data) != "new" { + t.Fatalf("replacement contents = %q, error = %v", data, err) + } + if _, err := os.Stat(source); !os.IsNotExist(err) { + t.Fatalf("source still present: %v", err) + } +} + +func TestWindowsReplaceMissingSource(t *testing.T) { + dir := t.TempDir() + target := filepath.Join(dir, "state") + if err := os.WriteFile(target, []byte("old"), 0600); err != nil { + t.Fatal(err) + } + if err := replaceFile(filepath.Join(dir, "missing"), target); err != windows.ERROR_FILE_NOT_FOUND { + t.Fatalf("missing source: %v", err) + } + data, err := os.ReadFile(target) + if err != nil || string(data) != "old" { + t.Fatalf("original contents = %q, error = %v", data, err) + } +} From a2882f60fd300f8d5cf40b1f99eb1f968f8e2c26 Mon Sep 17 00:00:00 2001 From: Pyuyi <204892921+VeraPyuyi@users.noreply.github.com> Date: Mon, 19 Oct 2026 15:49:07 +0800 Subject: [PATCH 31/47] test: synchronize capacity fixtures and retain child diagnostics Use atomic pause/revoke flags in capacity status fixtures. Capture event-store child stdout/stderr and wait for every child before reporting failures, preserving useful diagnostics under full-suite load. Capacity fixtures passed three race iterations in 26.67s; event cross-process diagnostics passed twenty race iterations before the separately committed Windows replacement fix. Latest full-suite uninstall timeout remains under investigation. --- internal/dotbridge/status_capacity_test.go | 11 ++++++----- internal/dotevents/store_test.go | 11 +++++++++-- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/internal/dotbridge/status_capacity_test.go b/internal/dotbridge/status_capacity_test.go index 09a1cd8..b45ac1b 100644 --- a/internal/dotbridge/status_capacity_test.go +++ b/internal/dotbridge/status_capacity_test.go @@ -6,6 +6,7 @@ import ( "net/http" "net/http/httptest" "strings" + "sync/atomic" "testing" "time" @@ -132,9 +133,9 @@ func TestCapacityPublicationAccessChanges(t *testing.T) { for _, change := range []string{"none", "disabled_before", "disable", "disable_enable", "revoke", "pause", "access_revoke"} { t.Run(fmt.Sprintf("admin=%v/%s", admin, change), func(t *testing.T) { s, _, id := openTestBridge(t) - var paused, revoked bool - s.paused = func() bool { return paused } - s.revoked = func() bool { return revoked } + var paused, revoked atomic.Bool + s.paused = paused.Load + s.revoked = revoked.Load q := &capacityInterceptQueue{QueueAPI: s.queue} s.queue = q q.after = func() { @@ -153,9 +154,9 @@ func TestCapacityPublicationAccessChanges(t *testing.T) { t.Fatal(err) } case "pause": - paused = true + paused.Store(true) case "access_revoke": - revoked = true + revoked.Store(true) } } if change == "disabled_before" { diff --git a/internal/dotevents/store_test.go b/internal/dotevents/store_test.go index 4dfba49..b7cd30b 100644 --- a/internal/dotevents/store_test.go +++ b/internal/dotevents/store_test.go @@ -1,6 +1,7 @@ package dotevents import ( + "bytes" "os" "os/exec" "path/filepath" @@ -28,19 +29,25 @@ func TestCrossProcessLockAndMissingState(t *testing.T) { t.Fatal(e) } cmds := make([]*exec.Cmd, 3) + outputs := make([]bytes.Buffer, len(cmds)) for i := range cmds { c := exec.Command(exe, "-test.run=^TestProcessLockHelper$") + c.Stdout = &outputs[i] + c.Stderr = &outputs[i] c.Env = append(os.Environ(), "DOT_EVENTS_TEST_HELPER=1", "DOT_EVENTS_TEST_DIR="+dir) cmds[i] = c if e = c.Start(); e != nil { t.Fatal(e) } } - for _, c := range cmds { + for i, c := range cmds { if e = c.Wait(); e != nil { - t.Fatal(e) + t.Errorf("child %d: %v\n%s", i, e, outputs[i].String()) } } + if t.Failed() { + t.FailNow() + } if _, e = m.Status("alice", "alice"); e != nil { t.Fatal(e) } From 3549c5fb2b6d7cd33d6767bfef035af5d7b374ba Mon Sep 17 00:00:00 2001 From: Pyuyi <204892921+VeraPyuyi@users.noreply.github.com> Date: Mon, 19 Oct 2026 15:50:08 +0800 Subject: [PATCH 32/47] docs: qualify loaded event diagnostic log coverage --- externalresearch/dot-events-concurrency-diagnosis.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/externalresearch/dot-events-concurrency-diagnosis.md b/externalresearch/dot-events-concurrency-diagnosis.md index 5850ca9..1db57ff 100644 --- a/externalresearch/dot-events-concurrency-diagnosis.md +++ b/externalresearch/dot-events-concurrency-diagnosis.md @@ -9,7 +9,7 @@ The reproduced failures originate in `store.saveState` when Windows refuses the Evidence is retained in `C:/Users/13678/AppData/Local/Temp/opencode/magpie-context-verification/`: - `dot-events-flake-focused-diagnostic-02.log`: 100 repetitions of both originally failing tests under race. Two cross-process failures and two concurrent-subscription failures reproduce. Every captured unknown stack points to `saveState`'s replacement branch (original store.go:168); subscription fails at its first persistence transaction (manager.go:61), before callback verification / generation finalization. Package fails after 149.619s. -- `dot-events-flake-loaded-diagnostic-03.log`: same target tests alongside queue/bridge concurrency/capacity tests, 100 repetitions. Four captured child failures explicitly report `windows_move_file_ex: Access is denied`, followed by `saveState` stacks. Events finishes in 150.408s; other package completion is recorded separately in this log. +- `dot-events-flake-loaded-diagnostic-03.log`: same target tests alongside queue/bridge concurrency/capacity tests, 100 repetitions. Four captured child failures explicitly report `windows_move_file_ex: Access is denied`, followed by `saveState` stacks. Events finishes in 150.408s. This log does not retain queue/bridge completion results, so it substantiates the captured events failures only. - `dot-events-flake-acl-isolation-04.log`: an investigation-only test repeatedly replaces a file while a second goroutine reapplies its parent directory ACL. All three repetitions fail on replacement iterations 1, 1, 3 with access denied. - `dot-events-flake-acl-protected-isolation-05.log`: repeat with protected current-user ACLs on source and target, matching store privacy setup. All three repetitions fail on iterations 2, 0, 3. This isolates ACL propagation as a sufficient trigger, without HTTP, manager generations, or flock contention. - `dot-events-flake-regression-red-06.log`: committed regression test holds the target with a real Windows handle allowing read/write sharing but denying delete sharing; closes it after 50ms. Existing single-attempt replacement fails with access denied. The missing-source test also checks that the original target remains intact. From 34ba3c5800ba1efd267490664c6ce16aa2680ac8 Mon Sep 17 00:00:00 2001 From: Pyuyi <204892921+VeraPyuyi@users.noreply.github.com> Date: Mon, 19 Oct 2026 16:15:34 +0800 Subject: [PATCH 33/47] fix(desktop): reject stale Windows parent PID relationships --- cmd/dotcontrol/install_exit_test.go | 58 +++++++++++-- .../dot-uninstall-wait-diagnosis.md | 86 +++++++++++++++++++ internal/desktop/process_tree_test.go | 74 ++++++++++++++++ internal/desktop/process_tree_windows_test.go | 33 +++++++ internal/desktop/references.go | 32 +++++-- internal/desktop/references_windows.go | 4 + 6 files changed, 273 insertions(+), 14 deletions(-) create mode 100644 externalresearch/dot-uninstall-wait-diagnosis.md create mode 100644 internal/desktop/process_tree_test.go create mode 100644 internal/desktop/process_tree_windows_test.go diff --git a/cmd/dotcontrol/install_exit_test.go b/cmd/dotcontrol/install_exit_test.go index 7632e8d..8d05310 100644 --- a/cmd/dotcontrol/install_exit_test.go +++ b/cmd/dotcontrol/install_exit_test.go @@ -1,8 +1,10 @@ package main import ( + "bytes" "context" "encoding/json" + "fmt" "github.com/dotcontrol/dotcontrol/internal/config" "github.com/dotcontrol/dotcontrol/internal/desktop" "github.com/dotcontrol/dotcontrol/internal/vault" @@ -22,15 +24,21 @@ func TestUninstallExitFixture(t *testing.T) { for i, arg := range os.Args { if arg == "--uninstall-exit-fixture" && i+1 < len(os.Args) { dir := os.Args[i+1] - os.WriteFile(filepath.Join(dir, "ready"), []byte("ready"), 0600) + if err := os.WriteFile(filepath.Join(dir, "ready"), []byte("ready"), 0600); err != nil { + t.Fatal(err) + } + fmt.Fprintln(os.Stdout, "fixture ready") deadline := time.Now().Add(10 * time.Second) for time.Now().Before(deadline) { if _, e := os.Stat(filepath.Join(dir, "exit")); e == nil { + fmt.Fprintln(os.Stdout, "fixture received exit request") time.Sleep(500 * time.Millisecond) + fmt.Fprintln(os.Stdout, "fixture exiting") os.Exit(0) } time.Sleep(10 * time.Millisecond) } + fmt.Fprintln(os.Stderr, "fixture timed out waiting for exit request") os.Exit(1) } } @@ -38,15 +46,42 @@ func TestUninstallExitFixture(t *testing.T) { func TestUninstallWaitsForProcessAfterEngineLockRelease(t *testing.T) { keyring.MockInit() state := t.TempDir() - exe, _ := os.Executable() + exe, err := os.Executable() + if err != nil { + t.Fatal(err) + } cmd := exec.Command(exe, "-test.run=^TestUninstallExitFixture$", "--", "--uninstall-exit-fixture", state) + var output bytes.Buffer + cmd.Stdout = &output + cmd.Stderr = &output desktop.HideConsole(cmd) if e := cmd.Start(); e != nil { t.Fatal(e) } - done := make(chan error, 1) - go func() { done <- cmd.Wait() }() - defer os.WriteFile(filepath.Join(state, "exit"), []byte("exit"), 0600) + done := make(chan struct{}) + var waitErr error + go func() { + waitErr = cmd.Wait() + close(done) + }() + t.Cleanup(func() { + if err := os.WriteFile(filepath.Join(state, "exit"), []byte("exit"), 0600); err != nil { + t.Error(err) + } + select { + case <-done: + case <-time.After(5 * time.Second): + t.Errorf("fixture %d did not exit during cleanup", cmd.Process.Pid) + // Only the child created by this test may be terminated. + if err := cmd.Process.Kill(); err != nil { + t.Logf("fixture cleanup kill: %v", err) + } + <-done + } + if waitErr != nil || t.Failed() { + t.Logf("fixture pid=%d wait=%v output:\n%s", cmd.Process.Pid, waitErr, output.String()) + } + }) deadline := time.Now().Add(5 * time.Second) for { if _, e := os.Stat(filepath.Join(state, "ready")); e == nil { @@ -70,8 +105,12 @@ func TestUninstallWaitsForProcessAfterEngineLockRelease(t *testing.T) { return } if r.URL.Path == "/admin/shutdown" { - lock.Unlock() - os.WriteFile(filepath.Join(state, "exit"), []byte("exit"), 0600) + if err := lock.Unlock(); err != nil { + t.Error(err) + } + if err := os.WriteFile(filepath.Join(state, "exit"), []byte("exit"), 0600); err != nil { + t.Error(err) + } return } http.NotFound(w, r) @@ -91,7 +130,8 @@ func TestUninstallWaitsForProcessAfterEngineLockRelease(t *testing.T) { if time.Since(start) < 500*time.Millisecond { t.Fatal("uninstall proceeded at engine release while process was still alive") } - if e = <-done; e != nil { - t.Fatal(e) + <-done + if waitErr != nil { + t.Fatal(waitErr) } } diff --git a/externalresearch/dot-uninstall-wait-diagnosis.md b/externalresearch/dot-uninstall-wait-diagnosis.md new file mode 100644 index 0000000..8f64182 --- /dev/null +++ b/externalresearch/dot-uninstall-wait-diagnosis.md @@ -0,0 +1,86 @@ +# Windows uninstall process-wait diagnosis + +Investigated 2026-10-19 in `D:/Applications/Opencode/data/share/worktree/bd2540/dot-task-event-bridge`, starting at `9705d8f`. The shared worktree advanced to `3549c5f` while checks ran; the controller committed its existing capacity synchronization and events child diagnostics as `a2882f6`. This worker did not edit events or bridge files, live configuration, installed services, startup registrations, or credentials. All fixture credentials used `keyring.MockInit`. No subagents were used, and no unrelated machine process was signaled. + +## Windows references + +- [PROCESSENTRY32](https://learn.microsoft.com/en-us/windows/win32/api/tlhelp32/ns-tlhelp32-processentry32): parent field is the identifier of the creator process. +- [Why is CreateToolhelp32Snapshot returning incorrect parent process IDs?](https://devblogs.microsoft.com/oldnewthing/20150403-00/?p=44313): explains stale parent IDs after reuse and comparing creation times to reject an impossible parent relationship. +- [When does a process ID become available for reuse?](https://devblogs.microsoft.com/oldnewthing/20110107-00/?p=11803): explains process-object and handle lifetime, used when examining the fixture's `cmd.Wait` behavior. + +## Root cause and direct reproduction + +`dot-events-flake-full-fixed-09.log` originally failed `TestUninstallWaitsForProcessAfterEngineLockRelease` at `install_exit_test.go:89` after 30.11 seconds with `context deadline exceeded`. The fixture should exit 500ms after receiving its exit file, and has an independent ten-second watchdog. Extending the shutdown timeout would not explain or fix this failure. + +The reproduced bug is in process-tree descendant discovery, not the fixture's exit delay. Windows `PROCESSENTRY32.th32ParentProcessID` retains the original creator PID after that creator exits. A later process can reuse that PID. The traversal previously treated any parent-PID equality as proof of descent, recursively, and retained the resulting PIDs indefinitely. + +`dot-uninstall-minimal-diagnostic-06.log` reproduces the same 30.11-second failure twice in 100 race repetitions. Its first failure records: + +```text +stop begin pid=17540 at=2026-10-19T16:00:05.5518654+08:00 +observe begin pid=17540 +tree add pid=19152 parent=17540 name=conhost.exe +tree add pid=31488 parent=19152 name=OneDrive.Sync.Service.exe +observed ids=[17540 19152 31488] +shutdown unlocking pid=17540 at=2026-10-19T16:00:05.5752782+08:00 +wait end ids=[17540 19152 31488] err=context deadline exceeded +child pid=17540 wait= +child received exit at=2026-10-19T16:00:05.5835182+08:00 +child exiting at=2026-10-19T16:00:06.0841138+08:00 +``` + +A read-only `Get-Process -Id 31488 | Select-Object Id,ProcessName,StartTime` showed `OneDrive.Sync.Service` started on **2026-10-18 at 16:28:08**, almost a day before the fixture's console host. It cannot be that newly created host's child. The second failure in the same log again adopts PID 31488 through a new host that reused PID 19152. The unrelated process was never stopped. The test child exited with success and was waited for in both failures. + +Investigation overlays captured child output, wait completion, adopted PIDs and, in a separate diagnostic variant, Windows creation times. The reproducing minimal variant kept the original nil stdout/stderr streams and wrote its child diagnostics inside the fixture's temporary directory. Repository production sources were not instrumented. Every fixture cleanup requested exit and waited; its fallback termination was restricted to the exact child created by that test. + +## Baseline proof and deterministic regression + +The following sources are byte-for-byte equivalent after newline normalization at `4208e05` and `9705d8f`: `cmd/dotcontrol/install_exit_test.go`, `cmd/dotcontrol/install.go`, `internal/desktop/references.go`, and `internal/desktop/references_windows.go`. A `git archive` copy of `4208e05` was extracted outside the worktree for comparisons. + +The deterministic regression models the observed process table: root 17540 created at time 200, console host 19152 at 300, and unrelated process 31488 at 100 with its stale parent PID still 19152. It also checks the next snapshot after the owned processes disappear, so merely filtering the initial snapshot cannot satisfy it. + +- `dot-uninstall-regression-red-08.log`: the current traversal fails, adopting `[17540 19152 31488]`. +- `dot-uninstall-base-regression-red-09.log`: the same fixture fails identically on `4208e05`. The overlay only extracts the existing traversal into a callable step and adds an unused creation-time field; it changes no traversal decision. +- `dot-uninstall-base-final-red-18.log`: the final regression fixtures also fail on the base's unchanged traversal and snapshot reader, including the native snapshot age check. The overlay adds only the extraction, unused metadata fields and fixtures. +- `dot-uninstall-base-final-green-17.log`: overlaying the final helper and snapshot changes onto that same base passes all final process-tree and native snapshot regressions for 30 race repetitions. + +This proves a preexisting process-discovery bug. It is not a basis for waiving the original full-suite failure; the bug is fixed here. The historical log lacked process diagnostics, so the exact PID adopted in that historical invocation is unknown. The reproductions match its test, deadline, and error, and directly identify the faulty operation. + +## Fix and preserved behavior + +Windows process snapshots now obtain creation `FILETIME` using `GetProcessTimes` on the existing `PROCESS_QUERY_LIMITED_INFORMATION` handle. Live observation retains creation evidence for each adopted process and rejects a proposed child only when both times are known and the child predates its purported parent. Evidence survives later snapshots after parent exit, preventing delayed adoption of the same stale relationship. + +Unknown creation times continue to block conservatively. Genuine descendants, equal timestamps, orphaned descendants, and occupied previously recorded PIDs still block exit. `WaitProcessIDs` recovery records contain only PIDs; creation filtering is disabled for that recovery path, preserving its original conservative interpretation of uncertain ancestry. A separate regression caught and prevented weakening recovery during implementation. + +The same traversal still watches descendants accepted while shutdown drains. There is no timeout increase, forced production termination, process-name exemption, PID removal, or change to shutdown endpoints. Unix snapshot creation values stay zero, leaving Unix traversal behavior unchanged. This narrowly changes the proven false-descendant case in the shared stop helper used by uninstall and upgrade; public shutdown behavior otherwise stays as before. + +`install_exit_test.go` now checks fixture file writes, captures stdout/stderr, and always waits for its child during cleanup before inspecting output or allowing temporary-directory cleanup. Failed runs report the child PID, wait result and output. The existing 500ms delay and production 30-second shutdown deadline remain unchanged. + +## Verification + +All logs and runner scripts are under `C:/Users/13678/AppData/Local/Temp/opencode/magpie-context-verification/`. Commands use the existing `run_checks.py --race` sandbox, `C:/Program Files/Go/bin/go.exe` (Go 1.27), `CGO_ENABLED=1`, and `CC=C:/msys64/mingw64/bin/gcc.exe`, with that compiler directory on PATH. HOME, APPDATA, USERPROFILE and temporary directories are isolated while the existing Go caches are retained. + +Final checks: + +```text +go test -race ./internal/desktop ./cmd/dotcontrol -run '^TestProcessTree|^TestWindowsProcessSnapshot|^TestUninstallWaitsForProcessAfterEngineLockRelease$' -count=30 -timeout=5m +go test -race ./... -count=1 -timeout=10m +go vet ./internal/desktop ./cmd/dotcontrol +``` + +- `dot-uninstall-final-focused-12.log`: passes all focused checks for 30 repetitions (desktop 2.548s, CLI 58.362s; runner 62.31s). +- `dot-uninstall-full-final-13.log`: **full repository race suite passes**, exit 0, runner 65.46s. CLI 13.391s, desktop 4.282s, bridge 50.452s, events 6.597s, queue 47.762s. +- `dot-uninstall-vet-14.log`: targeted vet passes, exit 0. +- `dot-uninstall-final-loaded-15.log`: **all packages pass** for 100 race repetitions with the original fixture's nil streams and final production source. Concurrent packages are CLI (178.331s), queue (58.279s), and events (151.644s); runner 183.24s. Command: `go test -race -overlay=.../uninstall-diagnosis/final-load-overlay.json ./cmd/dotcontrol ./internal/dotqueue ./internal/dotevents -run '^TestUninstallWaitsForProcessAfterEngineLockRelease$|^TestConcurrentSubmitAndClaim$|^TestConcurrentSubscribeAndDeliver$|^TestCrossProcessLockAndMissingState$' -count=100 -timeout=10m`. +- `gofmt` and `git diff --check` pass for the final changes. + +Other completed investigation runs and limitations: + +- Original focused fixture: 20 race repetitions passed (`dot-uninstall-original-focus-01.log`). Captured-output diagnostics: 50 fixture repetitions passed alongside package load (`dot-uninstall-diagnostic-load-02.log`); 50 baseline fixture repetitions passed (`dot-uninstall-base-diagnostic-03.log`); four concurrent test processes each passed 50 repetitions (`dot-uninstall-stress-05.log` and per-worker logs). These passes did not establish absence of the bug. The minimal variant subsequently reproduced it twice. +- `dot-uninstall-handle-probe-07.log` retains an additional handle to the owned child; the wait passes. That experiment did not support the theory that retaining an exited process handle by itself blocks the process-table wait. +- `dot-uninstall-diagnostic-load-02.log` is **not an overall green run**: the queue package reached the total 15-minute budget during repeated capacity fixtures. Its stack is executing JSON parsing in `fillStoreJobs` for `TestCapacityQueueFullAcrossClients`; no queue behavior conclusion is drawn. CLI, bridge and events completed successfully. +- `dot-uninstall-full-diagnostic-04.log` is **red**: all three uninstall repetitions pass, but `TestWriteAfterPauseStaysConservative` reports an actual `DATA RACE`. The write is `internal/dotbridge/http_budget_test.go:200`; the concurrent read is `Service.gate` at `service.go:140`, via the worker authorization path. Bridge files are outside this worker's ownership and were left untouched. The final single-count full suite is green, but that does not erase or waive this intermittent race finding; the controller has a concrete trace to address. +- `dot-uninstall-recovery-red-11.log` intentionally fails the additional recovery constraint before restricting creation filtering to live observation. The final focused run includes the passing recovery regression. +- `dot-uninstall-base-final-red-16.log` is a build failure from a stale diagnostic overlay lacking the new unused map field; it is not behavioral red evidence. The earlier base regression red is valid. + +No race checks were disabled or waived. No non-Windows tests were executed; the shared traversal's Unix behavior is preserved by unavailable creation metadata. diff --git a/internal/desktop/process_tree_test.go b/internal/desktop/process_tree_test.go new file mode 100644 index 0000000..4c123c0 --- /dev/null +++ b/internal/desktop/process_tree_test.go @@ -0,0 +1,74 @@ +package desktop + +import ( + "reflect" + "testing" +) + +func TestProcessTreeRejectsChildOlderThanReusedParentPID(t *testing.T) { + // Windows leaves a child's original parent PID in the process table after + // that parent exits. A new console host can reuse it much later. + tree := &ProcessTree{ids: map[int]bool{17540: true}, created: map[int]uint64{}} + if !tree.remainingProcesses([]programProcess{ + {PID: 31488, ParentPID: 19152, Created: 100}, + {PID: 19152, ParentPID: 17540, Created: 300}, + {PID: 17540, ParentPID: 1, Created: 200}, + }) { + t.Fatal("live agent was not retained") + } + if got := tree.PIDs(); !reflect.DeepEqual(got, []int{17540, 19152}) { + t.Fatalf("adopted a process older than its purported parent: %v", got) + } + // The creation evidence must survive the parent's exit, or the next poll + // would adopt the same unrelated process and wait until the stop deadline. + if tree.remainingProcesses([]programProcess{{PID: 31488, ParentPID: 19152, Created: 100}}) { + t.Fatal("unrelated old process blocked exit after the owned tree exited") + } +} + +func TestProcessTreePersistedIDsKeepUncertainAncestry(t *testing.T) { + // Recovery records contain only PIDs, not the original creation times. The + // current PID occupant may be newer than a genuine, now-orphaned child. + tree := &ProcessTree{ids: map[int]bool{10: true}} + tree.remainingProcesses([]programProcess{ + {PID: 10, Created: 300}, + {PID: 20, ParentPID: 10, Created: 100}, + }) + if !tree.remainingProcesses([]programProcess{{PID: 20, ParentPID: 10, Created: 100}}) { + t.Fatal("recovery discarded uncertain descendant evidence") + } +} + +func TestProcessTreeKeepsDescendantsAndUnknownCreationEvidence(t *testing.T) { + for _, tc := range []struct { + name string + parent, child uint64 + }{ + {"newer child", 200, 300}, + {"same timestamp", 200, 200}, + {"unknown child", 200, 0}, + {"unknown parent", 0, 100}, + } { + t.Run(tc.name, func(t *testing.T) { + tree := &ProcessTree{ids: map[int]bool{10: true}, created: map[int]uint64{}} + tree.remainingProcesses([]programProcess{ + {PID: 30, ParentPID: 20, Created: tc.child}, + {PID: 20, ParentPID: 10, Created: tc.parent}, + {PID: 10}, + }) + if got := tree.PIDs(); !reflect.DeepEqual(got, []int{10, 20, 30}) { + t.Fatalf("lost conservative descendant evidence: %v", got) + } + if !tree.remainingProcesses([]programProcess{{PID: 30, ParentPID: 20, Created: tc.child}}) { + t.Fatal("orphaned descendant no longer blocks exit") + } + if tree.remainingProcesses(nil) { + t.Fatal("empty process table still blocks exit") + } + // Persisted PID evidence deliberately remains conservative under reuse. + if !tree.remainingProcesses([]programProcess{{PID: 30, Created: 900}}) { + t.Fatal("occupied recorded PID no longer blocks exit") + } + }) + } +} diff --git a/internal/desktop/process_tree_windows_test.go b/internal/desktop/process_tree_windows_test.go new file mode 100644 index 0000000..eba0653 --- /dev/null +++ b/internal/desktop/process_tree_windows_test.go @@ -0,0 +1,33 @@ +//go:build windows + +package desktop + +import ( + "context" + "os" + "testing" +) + +func TestWindowsProcessSnapshotSupportsParentAgeCheck(t *testing.T) { + processes, err := listProgramProcesses(context.Background()) + if err != nil { + t.Fatal(err) + } + for _, process := range processes { + if process.PID != os.Getpid() { + continue + } + // A child dating from the Windows epoch cannot belong to this test + // process. Exercise the actual snapshot metadata used by the waiter. + tree := &ProcessTree{ids: map[int]bool{process.PID: true}, created: map[int]uint64{}} + tree.remainingProcesses([]programProcess{ + {PID: -1, ParentPID: process.PID, Created: 1}, + process, + }) + if ids := tree.PIDs(); len(ids) != 1 || ids[0] != process.PID { + t.Fatalf("Windows snapshot could not reject an older child: %v", ids) + } + return + } + t.Fatal("test process absent from Windows snapshot") +} diff --git a/internal/desktop/references.go b/internal/desktop/references.go index ca291f1..2a50701 100644 --- a/internal/desktop/references.go +++ b/internal/desktop/references.go @@ -32,6 +32,7 @@ type StartupSnapshot struct { type programProcess struct { PID, ParentPID int Name, Executable string + Created uint64 // Windows creation FILETIME; zero means unavailable. } func pathInside(root, p string) bool { @@ -137,15 +138,16 @@ func WaitExecutableExit(ctx context.Context, paths []string) error { // ProcessTree remembers the agent and every observed descendant. Keeping the // PIDs until absent is conservative under PID reuse and cannot stop another app. type ProcessTree struct { - mu sync.Mutex - ids map[int]bool + mu sync.Mutex + ids map[int]bool + created map[int]uint64 // nil for persisted PID evidence with unknown original ages. } func ObserveProcessTree(ctx context.Context, pid int) (*ProcessTree, error) { if pid <= 0 { return nil, errors.New("agent_process_identity_unverified") } - tree := &ProcessTree{ids: map[int]bool{pid: true}} + tree := &ProcessTree{ids: map[int]bool{pid: true}, created: map[int]uint64{}} if _, err := tree.remaining(ctx); err != nil { return nil, err } @@ -158,21 +160,41 @@ func (t *ProcessTree) remaining(ctx context.Context) (bool, error) { if err != nil { return true, err } + return t.remainingProcesses(processes), nil +} + +// remainingProcesses is called with t.mu held. +func (t *ProcessTree) remainingProcesses(processes []programProcess) bool { + for _, p := range processes { + if t.created != nil && t.ids[p.PID] && t.created[p.PID] == 0 && p.Created != 0 { + t.created[p.PID] = p.Created + } + } for changed := true; changed; { changed = false for _, p := range processes { if t.ids[p.ParentPID] && !t.ids[p.PID] { + // Windows retains a child's original parent PID even after that + // parent exits and its PID is reused. A child older than the + // observed parent cannot belong to it. Retain creation evidence + // across polls so the stale relationship stays excluded after exit. + if parent := t.created[p.ParentPID]; parent != 0 && p.Created != 0 && p.Created < parent { + continue + } t.ids[p.PID] = true + if t.created != nil { + t.created[p.PID] = p.Created + } changed = true } } } for _, p := range processes { if t.ids[p.PID] { - return true, nil + return true } } - return false, nil + return false } // Watch records descendants throughout draining, including children created by diff --git a/internal/desktop/references_windows.go b/internal/desktop/references_windows.go index 14289aa..4966c8f 100644 --- a/internal/desktop/references_windows.go +++ b/internal/desktop/references_windows.go @@ -116,6 +116,10 @@ func listProgramProcesses(ctx context.Context) ([]programProcess, error) { p := programProcess{PID: int(entry.ProcessID), ParentPID: int(entry.ParentProcessID), Name: windows.UTF16ToString(entry.ExeFile[:])} h, err := windows.OpenProcess(windows.PROCESS_QUERY_LIMITED_INFORMATION, false, entry.ProcessID) if err == nil { + var created, exited, kernel, user windows.Filetime + if windows.GetProcessTimes(h, &created, &exited, &kernel, &user) == nil { + p.Created = uint64(created.HighDateTime)<<32 | uint64(created.LowDateTime) + } buf := make([]uint16, 32768) n := uint32(len(buf)) if windows.QueryFullProcessImageName(h, 0, &buf[0], &n) == nil { From 9db5f33c85cd6757db55826470500f795d20e28d Mon Sep 17 00:00:00 2001 From: Pyuyi <204892921+VeraPyuyi@users.noreply.github.com> Date: Mon, 19 Oct 2026 16:19:13 +0800 Subject: [PATCH 34/47] test(dotbridge): install lifecycle callbacks before worker startup Remove post-Open callback replacement from pause/revoke fixtures. Configure immutable lifecycle callbacks before the service starts its worker, retaining atomic flag transitions for test interleavings. The observed callback-pointer DATA RACE is addressed across all matching fixtures. Focused write-after-pause race count20 passed; complete repository go test -race -count=1 -timeout 3m ./... passed in 59.7 seconds, retained dotcontrol-startup-gates-final-race.log. --- internal/dotbridge/bridge_test.go | 15 +++++++++++++-- internal/dotbridge/http_budget_test.go | 10 ++++------ internal/dotbridge/list_api_test.go | 3 +-- internal/dotbridge/review_test.go | 5 ++--- internal/dotbridge/status_capacity_test.go | 4 +--- 5 files changed, 21 insertions(+), 16 deletions(-) diff --git a/internal/dotbridge/bridge_test.go b/internal/dotbridge/bridge_test.go index e2f679a..5f007f4 100644 --- a/internal/dotbridge/bridge_test.go +++ b/internal/dotbridge/bridge_test.go @@ -20,6 +20,17 @@ func (noopBackend) Call(context.Context, string, service.Request) (any, error) { } func openTestBridge(t *testing.T) (*Service, string, string) { + t.Helper() + return openTestBridgeWithPause(t, func() bool { return false }) +} + +// Set lifecycle callbacks before Open starts the background worker. +func openTestBridgeWithPause(t *testing.T, paused func() bool) (*Service, string, string) { + t.Helper() + return openTestBridgeWithGates(t, paused, func() bool { return false }) +} + +func openTestBridgeWithGates(t *testing.T, paused, revoked func() bool) (*Service, string, string) { t.Helper() root := t.TempDir() vault := NewFakeVault() @@ -36,8 +47,8 @@ func openTestBridge(t *testing.T) (*Service, string, string) { Vault: vault, MCPQueue: "client.a", EventsDelivery: delivery, - Paused: func() bool { return false }, - Revoked: func() bool { return false }, + Paused: paused, + Revoked: revoked, }) if err != nil { t.Fatal(err) diff --git a/internal/dotbridge/http_budget_test.go b/internal/dotbridge/http_budget_test.go index a240f80..19263c9 100644 --- a/internal/dotbridge/http_budget_test.go +++ b/internal/dotbridge/http_budget_test.go @@ -192,12 +192,11 @@ func (q *writeAfterAccessQueue) Submit(clientID string, sub dotqueue.Submission) // TestWriteAfterPauseStaysConservative proves a durable write followed by a // pause is never reported accepted and the original job survives unchanged. func TestWriteAfterPauseStaysConservative(t *testing.T) { - s, token, id := openTestBridge(t) + var paused atomic.Bool + s, token, id := openTestBridgeWithPause(t, paused.Load) readyHTTPClient(t, s, id) api := httptest.NewServer(s.Handler()) defer api.Close() - var paused atomic.Bool - s.paused = paused.Load s.queue = &writeAfterAccessQueue{QueueAPI: s.queue, mutate: func() { paused.Store(true) }} code, raw, _ := submitReceipt(t, api, token, []byte(`{"request_id":"write.pause","mode":"answer","session_id":"chat","messages":[{"role":"user","content":"hello"}]}`)) if code == http.StatusAccepted || bytes.Contains(raw, []byte("accepted")) { @@ -249,12 +248,11 @@ func TestListErrorPrivacyAndGatePrecedence(t *testing.T) { const secret = "vault-secret-material" for _, condition := range []string{"gate_after", "gate_before"} { t.Run(condition, func(t *testing.T) { - s, token, id := openTestBridge(t) + var paused atomic.Bool + s, token, id := openTestBridgeWithPause(t, paused.Load) if _, err := s.queue.Submit(id, dotqueue.Submission{RequestID: "leak.probe", Mode: dotqueue.ModeAnswer, Messages: []dotqueue.Message{{Role: "user", Content: "private prompt"}}}); err != nil { t.Fatal(err) } - var paused atomic.Bool - s.paused = paused.Load s.queue = &listErrorQueue{QueueAPI: s.queue, secret: secret, mutate: func() { paused.Store(true) }} if condition == "gate_before" { paused.Store(true) diff --git a/internal/dotbridge/list_api_test.go b/internal/dotbridge/list_api_test.go index 356b631..9e8e2fe 100644 --- a/internal/dotbridge/list_api_test.go +++ b/internal/dotbridge/list_api_test.go @@ -91,9 +91,8 @@ func TestListAPIOwnerMetadata(t *testing.T) { func TestListAPIGates(t *testing.T) { for _, condition := range []string{"missing_auth", "bad_auth", "revoked", "disabled", "paused", "paused_after", "disabled_after", "revoked_after"} { t.Run(condition, func(t *testing.T) { - s, token, id := openTestBridge(t) var paused atomic.Bool - s.paused = paused.Load + s, token, id := openTestBridgeWithPause(t, paused.Load) var calls atomic.Int32 alter := func() { switch strings.TrimSuffix(condition, "_after") { diff --git a/internal/dotbridge/review_test.go b/internal/dotbridge/review_test.go index eb70011..b57c67d 100644 --- a/internal/dotbridge/review_test.go +++ b/internal/dotbridge/review_test.go @@ -59,13 +59,12 @@ func TestReviewProbeCannotUndoRevoke(t *testing.T) { } } func TestReviewReadRechecksPause(t *testing.T) { - s, _, id := openTestBridge(t) + var paused atomic.Bool + s, _, id := openTestBridgeWithPause(t, paused.Load) p, e := s.StartProbe(id) if e != nil { t.Fatal(e) } - var paused atomic.Bool - s.paused = paused.Load s.queue = interceptQueue{QueueAPI: s.queue, after: func() { paused.Store(true) }} if _, e := s.getJob(ClientPolicy{ClientID: id}, p.JobID); e == nil { t.Fatal("read succeeded after pause during I/O") diff --git a/internal/dotbridge/status_capacity_test.go b/internal/dotbridge/status_capacity_test.go index b45ac1b..27df721 100644 --- a/internal/dotbridge/status_capacity_test.go +++ b/internal/dotbridge/status_capacity_test.go @@ -132,10 +132,8 @@ func TestCapacityPublicationAccessChanges(t *testing.T) { for _, admin := range []bool{false, true} { for _, change := range []string{"none", "disabled_before", "disable", "disable_enable", "revoke", "pause", "access_revoke"} { t.Run(fmt.Sprintf("admin=%v/%s", admin, change), func(t *testing.T) { - s, _, id := openTestBridge(t) var paused, revoked atomic.Bool - s.paused = paused.Load - s.revoked = revoked.Load + s, _, id := openTestBridgeWithGates(t, paused.Load, revoked.Load) q := &capacityInterceptQueue{QueueAPI: s.queue} s.queue = q q.after = func() { From 5d16f9856fb5643b809e8143c11ae59f8cfce433 Mon Sep 17 00:00:00 2001 From: Pyuyi <204892921+VeraPyuyi@users.noreply.github.com> Date: Tue, 20 Oct 2026 22:23:42 +0800 Subject: [PATCH 35/47] fix(mcp-events): publish official catalog and flat subscribe result events/list now returns delivery, inputSchema and payloadSchema from one shared catalog; disabled or unauthorized principals receive an empty array instead of null. events/subscribe returns the flat {id, refreshBefore, cursor, truncated} result required by the MCP Events wire instead of a nested subscription object. Verified with real HTTP handler tests and race runs over mcpserver, dotbridge and dotevents (39.93s). --- internal/dotbridge/events_swap.go | 16 ++++ internal/dotbridge/service.go | 6 +- internal/dotevents/delivery_swap.go | 10 ++ internal/dotevents/manager.go | 31 ++++++- internal/dotevents/types.go | 7 +- internal/mcpserver/bridge.go | 8 +- internal/mcpserver/events_gate_test.go | 57 ++++++++++++ .../mcpserver/events_subscribe_helpers.go | 64 +++++++++++++ .../mcpserver/events_subscribe_shape_test.go | 55 +++++++++++ internal/mcpserver/events_wire_test.go | 91 +++++++++++++++++++ 10 files changed, 336 insertions(+), 9 deletions(-) create mode 100644 internal/dotbridge/events_swap.go create mode 100644 internal/dotevents/delivery_swap.go create mode 100644 internal/mcpserver/events_gate_test.go create mode 100644 internal/mcpserver/events_subscribe_helpers.go create mode 100644 internal/mcpserver/events_subscribe_shape_test.go create mode 100644 internal/mcpserver/events_wire_test.go diff --git a/internal/dotbridge/events_swap.go b/internal/dotbridge/events_swap.go new file mode 100644 index 0000000..06b6223 --- /dev/null +++ b/internal/dotbridge/events_swap.go @@ -0,0 +1,16 @@ +package dotbridge + +import "github.com/dotcontrol/dotcontrol/internal/dotevents" + +// ReplaceEventsDeliveryForTest swaps the outbound event transport for offline +// integration tests only. +func (s *Service) ReplaceEventsDeliveryForTest(d *dotevents.Delivery) { + if d == nil { + d = dotevents.NewDelivery(nil) + } + a, ok := s.events.(eventsAdapter) + if !ok { + panic("events adapter unavailable") + } + a.Manager.SetDeliveryForTest(d) +} diff --git a/internal/dotbridge/service.go b/internal/dotbridge/service.go index 37a595f..39a27f7 100644 --- a/internal/dotbridge/service.go +++ b/internal/dotbridge/service.go @@ -361,10 +361,10 @@ func (s *Service) MCPPrincipal() (string, string, error) { } func (s *Service) EventsList(principal string) []dotevents.Definition { - if s.gate() != nil { - return nil + if s.authorizeQueue(principal) != nil { + return []dotevents.Definition{} } - return []dotevents.Definition{{Name: dotevents.EventName, Description: "Availability notices for an authorized paired queue."}} + return dotevents.Catalog() } func (s *Service) EventsSubscribe(ctx context.Context, principal string, params dotevents.Params) (dotevents.Subscription, error) { queue := params.Arguments.QueueID diff --git a/internal/dotevents/delivery_swap.go b/internal/dotevents/delivery_swap.go new file mode 100644 index 0000000..7932956 --- /dev/null +++ b/internal/dotevents/delivery_swap.go @@ -0,0 +1,10 @@ +package dotevents + +// SetDeliveryForTest replaces the outbound transport for offline integration +// tests. Only tests construct it with a fixture HTTP client. +func (m *Manager) SetDeliveryForTest(d *Delivery) { + if d == nil { + d = &Delivery{} + } + m.delivery = d +} diff --git a/internal/dotevents/manager.go b/internal/dotevents/manager.go index 8e92435..672f7c6 100644 --- a/internal/dotevents/manager.go +++ b/internal/dotevents/manager.go @@ -179,9 +179,36 @@ func (m *Manager) Unsubscribe(ctx context.Context, principal string, p Params) e } func (m *Manager) List(principal string) []Definition { if validateID(principal) != nil { - return nil + return []Definition{} } - return []Definition{{Name: EventName, Description: "A queued task is available; retrieve it using authorized queue tools."}} + return []Definition{Catalog()[0]} +} + +// Catalog returns the immutable discovery definition served over MCP Events. +func Catalog() []Definition { + return []Definition{{ + Name: EventName, + Description: "Availability notices for an authorized paired queue.", + Delivery: []string{"webhook"}, + InputSchema: map[string]any{ + "type": "object", + "properties": map[string]any{ + "queue_id": map[string]any{"type": "string", "description": "Authorized task queue ID."}, + }, + "required": []string{"queue_id"}, + "additionalProperties": false, + }, + PayloadSchema: map[string]any{ + "type": "object", + "properties": map[string]any{ + "queue_id": map[string]any{"type": "string"}, + "job_id": map[string]any{"type": "string"}, + "mode": map[string]any{"type": "string", "enum": []string{"answer", "task"}}, + }, + "required": []string{"queue_id", "job_id", "mode"}, + "additionalProperties": false, + }, + }} } func (m *Manager) Deliver(ctx context.Context, principal, queueID string, n Notice) error { if n.OccurredAt.IsZero() || n.OccurredAt.Year() < 1 || n.OccurredAt.Year() > 9999 || validateID(principal) != nil || principal != queueID || n.QueueID != queueID || validateID(n.EventID) != nil || validateID(n.JobID) != nil || (n.Mode != "answer" && n.Mode != "task") { diff --git a/internal/dotevents/types.go b/internal/dotevents/types.go index 62747a2..511fd4c 100644 --- a/internal/dotevents/types.go +++ b/internal/dotevents/types.go @@ -32,8 +32,11 @@ type Subscription struct { } type Definition struct { - Name string `json:"name"` - Description string `json:"description"` + Name string `json:"name"` + Description string `json:"description"` + Delivery []string `json:"delivery"` + InputSchema map[string]any `json:"inputSchema"` + PayloadSchema map[string]any `json:"payloadSchema"` } type Notice struct { diff --git a/internal/mcpserver/bridge.go b/internal/mcpserver/bridge.go index 81665ca..d055aed 100644 --- a/internal/mcpserver/bridge.go +++ b/internal/mcpserver/bridge.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "net/http" + "time" "github.com/dotcontrol/dotcontrol/internal/dotbridge" "github.com/dotcontrol/dotcontrol/internal/dotevents" @@ -57,7 +58,10 @@ type EventsSubscribeParams struct { type EventsSubscribeResult struct { mcp.ResultBase - Subscription dotevents.Subscription `json:"subscription"` + ID string `json:"id"` + RefreshBefore time.Time `json:"refreshBefore"` + Cursor any `json:"cursor"` + Truncated bool `json:"truncated"` } type EventsUnsubscribeParams struct { @@ -129,7 +133,7 @@ func NewWithBridge(backend service.Backend, bridge *dotbridge.Service) (*mcp.Ser if err != nil { return nil, err } - return &EventsSubscribeResult{Subscription: sub}, nil + return &EventsSubscribeResult{ID: sub.ID, RefreshBefore: sub.ExpiresAt, Cursor: nil, Truncated: false}, nil }); err != nil { return nil, err } diff --git a/internal/mcpserver/events_gate_test.go b/internal/mcpserver/events_gate_test.go new file mode 100644 index 0000000..c4c5ece --- /dev/null +++ b/internal/mcpserver/events_gate_test.go @@ -0,0 +1,57 @@ +package mcpserver + +import ( + "encoding/json" + "net/http/httptest" + "testing" +) + +// The events/list result keeps an array even when the bridge is disabled, and a +// revoked paired client is not offered an event catalog it cannot subscribe to. +func TestEventsListGateShapes(t *testing.T) { + metalist := func(t *testing.T) map[string]any { + return map[string]any{"_meta": map[string]any{ + "io.modelcontextprotocol/protocolVersion": "2026-07-28", + "io.modelcontextprotocol/clientCapabilities": map[string]any{}, + "io.modelcontextprotocol/clientInfo": map[string]any{"name": "fixture", "version": "1"}, + }} + } + s := openBridgeForMCPTest(t) + h, err := HTTPWithBridge(bridgeNoopBackend{}, s) + if err != nil { + t.Fatal(err) + } + server := httptest.NewServer(h) + defer server.Close() + env := eventsMCPRequest(t, server.URL, "events/list", metalist(t)) + var listed struct { + Events []map[string]json.RawMessage `json:"events"` + } + if json.Unmarshal(env["result"], &listed) != nil || len(listed.Events) != 1 { + t.Fatalf("enabled catalog: %s", env["result"]) + } + + if err := s.Disable(); err != nil { + t.Fatal(err) + } + env = eventsMCPRequest(t, server.URL, "events/list", metalist(t)) + var disabled struct { + Events json.RawMessage `json:"events"` + } + if json.Unmarshal(env["result"], &disabled) != nil { + t.Fatalf("disabled result: %s", env["result"]) + } + if string(disabled.Events) != "[]" { + t.Fatalf("disabled events must be an empty array: %s", disabled.Events) + } +} + +func TestEventsListExcludesRevokedClientCatalog(t *testing.T) { + s := openBridgeForMCPTest(t) + if err := s.RevokeClient("client.a"); err != nil { + t.Fatal(err) + } + if got := s.EventsList("client.a"); len(got) != 0 { + t.Fatalf("revoked client sees catalog: %#v", got) + } +} diff --git a/internal/mcpserver/events_subscribe_helpers.go b/internal/mcpserver/events_subscribe_helpers.go new file mode 100644 index 0000000..54161b6 --- /dev/null +++ b/internal/mcpserver/events_subscribe_helpers.go @@ -0,0 +1,64 @@ +package mcpserver + +import ( + "context" + "crypto/tls" + "crypto/x509" + "encoding/json" + "io" + "net" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/dotcontrol/dotcontrol/internal/dotevents" +) + +const fixtureCallbackURL = "https://callback.example/hook" + +// NewTestEventsDelivery provides the offline TLS callback used by the event +// wire tests. Production always uses the public HTTPS transport. +func NewTestEventsDelivery(t *testing.T) (*dotevents.Delivery, func()) { + t.Helper() + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(io.LimitReader(r.Body, 64*1024)) + var probe struct { + Type string `json:"type"` + Challenge string `json:"challenge"` + } + _ = json.Unmarshal(body, &probe) + if probe.Type == "verification" { + if probe.Challenge == "" { + w.WriteHeader(http.StatusBadRequest) + return + } + _ = json.NewEncoder(w).Encode(map[string]string{"challenge": probe.Challenge}) + return + } + if r.Header.Get("webhook-id") == "" || r.Header.Get("webhook-signature") == "" { + w.WriteHeader(http.StatusBadRequest) + return + } + w.WriteHeader(http.StatusOK) + })) + pool := x509.NewCertPool() + pool.AddCert(server.Certificate()) + tr := server.Client().Transport.(*http.Transport).Clone() + tr.Proxy = nil + tr.DisableKeepAlives = true + tr.TLSClientConfig = &tls.Config{RootCAs: pool, ServerName: "example.com"} //nolint:gosec // offline fixture + tr.DialContext = func(ctx context.Context, network, address string) (net.Conn, error) { + if address != "callback.example:443" { + return nil, &net.OpError{Op: "dial", Err: errFixtureDial} + } + return (&net.Dialer{Timeout: time.Second}).DialContext(ctx, network, server.Listener.Addr().String()) + } + return dotevents.NewDelivery(&http.Client{ + Timeout: 5 * time.Second, + Transport: tr, + CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }, + }), server.Close +} + +var errFixtureDial = &net.AddrError{Err: "fixture destination rejected", Addr: "callback"} diff --git a/internal/mcpserver/events_subscribe_shape_test.go b/internal/mcpserver/events_subscribe_shape_test.go new file mode 100644 index 0000000..a90d7ad --- /dev/null +++ b/internal/mcpserver/events_subscribe_shape_test.go @@ -0,0 +1,55 @@ +package mcpserver + +import ( + "encoding/json" + "net/http/httptest" + "testing" +) + +func TestEventsSubscribeOfficialResponseShapeOverHTTP(t *testing.T) { + s := openBridgeForMCPTest(t) + delivery, closeDelivery := NewTestEventsDelivery(t) + defer closeDelivery() + s.ReplaceEventsDeliveryForTest(delivery) + h, err := HTTPWithBridge(bridgeNoopBackend{}, s) + if err != nil { + t.Fatal(err) + } + server := httptest.NewServer(h) + defer server.Close() + env := eventsMCPRequest(t, server.URL, "events/subscribe", map[string]any{ + "_meta": map[string]any{ + "io.modelcontextprotocol/protocolVersion": "2026-07-28", + "io.modelcontextprotocol/clientCapabilities": map[string]any{}, + "io.modelcontextprotocol/clientInfo": map[string]any{"name": "fixture", "version": "1"}, + }, + "name": "dot.task.available", + "arguments": map[string]string{"queue_id": "client.a"}, + "delivery": map[string]string{ + "mode": "webhook", + "url": fixtureCallbackURL, + "secret": "whsec_MDEyMzQ1Njc4OWFiY2RlZjAxMjM0NTY3ODlhYmNkZWY=", + }, + "cursor": nil, + }) + var out map[string]json.RawMessage + if json.Unmarshal(env["result"], &out) != nil { + t.Fatalf("subscribe envelope: %s", env["result"]) + } + if out["id"] == nil || string(out["id"]) == `""` { + t.Fatalf("id missing: %s", env["result"]) + } + var refresh string + if json.Unmarshal(out["refreshBefore"], &refresh) != nil || refresh == "" { + t.Fatalf("refreshBefore missing: %s", env["result"]) + } + if string(out["cursor"]) != "null" || string(out["truncated"]) != "false" { + t.Fatalf("official cursor/truncated: %s", env["result"]) + } + if _, nested := out["subscription"]; nested { + t.Fatalf("nested subscription object: %s", env["result"]) + } + if _, leaked := out["secret"]; leaked { + t.Fatalf("secret leaked: %s", env["result"]) + } +} diff --git a/internal/mcpserver/events_wire_test.go b/internal/mcpserver/events_wire_test.go new file mode 100644 index 0000000..e4d407f --- /dev/null +++ b/internal/mcpserver/events_wire_test.go @@ -0,0 +1,91 @@ +package mcpserver + +import ( + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +func eventsMCPRequest(t *testing.T, url, method string, params any) map[string]json.RawMessage { + t.Helper() + payload, _ := json.Marshal(map[string]any{"jsonrpc": "2.0", "id": 1, "method": method, "params": params}) + req, _ := http.NewRequest(http.MethodPost, url, strings.NewReader(string(payload))) + req.Header.Set("Mcp-Method", method) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Accept", "application/json, text/event-stream") + req.Header.Set("MCP-Protocol-Version", "2026-07-28") + res, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + defer res.Body.Close() + raw, _ := io.ReadAll(res.Body) + if res.StatusCode != http.StatusOK { + t.Fatalf("%s: %d %s", method, res.StatusCode, raw) + } + var env map[string]json.RawMessage + if json.Unmarshal(raw, &env) != nil || env["error"] != nil { + t.Fatalf("%s: %s", method, raw) + } + return env +} + +func TestEventsListPublishesOfficialDefinition(t *testing.T) { + s := openBridgeForMCPTest(t) + h, err := HTTPWithBridge(bridgeNoopBackend{}, s) + if err != nil { + t.Fatal(err) + } + server := httptest.NewServer(h) + defer server.Close() + env := eventsMCPRequest(t, server.URL, "events/list", map[string]any{"_meta": map[string]any{"io.modelcontextprotocol/protocolVersion": "2026-07-28", "io.modelcontextprotocol/clientCapabilities": map[string]any{}, "io.modelcontextprotocol/clientInfo": map[string]any{"name": "fixture", "version": "1"}}}) + var result struct { + Events []map[string]json.RawMessage `json:"events"` + } + if json.Unmarshal(env["result"], &result) != nil || len(result.Events) != 1 { + t.Fatalf("list: %s", env["result"]) + } + event := result.Events[0] + if string(event["name"]) != `"dot.task.available"` { + t.Fatalf("name: %s", event["name"]) + } + var delivery []string + if json.Unmarshal(event["delivery"], &delivery) != nil || len(delivery) != 1 || delivery[0] != "webhook" { + t.Fatalf("delivery: %s", event["delivery"]) + } + for _, key := range []string{"inputSchema", "payloadSchema"} { + var schema map[string]json.RawMessage + if json.Unmarshal(event[key], &schema) != nil || schema["type"] == nil || schema["properties"] == nil { + t.Fatalf("%s: %s", key, event[key]) + } + } + var input struct { + Required []string `json:"required"` + Properties map[string]struct { + Type string `json:"type"` + } `json:"properties"` + } + if json.Unmarshal(event["inputSchema"], &input) != nil || len(input.Required) != 1 || input.Required[0] != "queue_id" || input.Properties["queue_id"].Type != "string" { + t.Fatalf("inputSchema: %s", event["inputSchema"]) + } + var payload struct { + Required []string `json:"required"` + Properties map[string]struct { + Type string `json:"type"` + } `json:"properties"` + } + if json.Unmarshal(event["payloadSchema"], &payload) != nil { + t.Fatalf("payloadSchema: %s", event["payloadSchema"]) + } + for _, name := range []string{"queue_id", "job_id", "mode"} { + if payload.Properties[name].Type != "string" { + t.Fatalf("payload field %s: %s", name, event["payloadSchema"]) + } + } + if strings.Contains(string(event["payloadSchema"]), "prompt") || strings.Contains(string(event["payloadSchema"]), "token") { + t.Fatalf("payload exposes private fields: %s", event["payloadSchema"]) + } +} From 537a2710275831ed20e871060e53a79eaf3b39d9 Mon Sep 17 00:00:00 2001 From: Pyuyi <204892921+VeraPyuyi@users.noreply.github.com> Date: Tue, 20 Oct 2026 22:51:13 +0800 Subject: [PATCH 36/47] fix(mcp-events): keep catalog discoverable before subscription events/list gated on per-queue authorization, so the authenticated MCP consumer received an empty catalog and could never subscribe. Discovery now requires only an enabled bridge; subscribe still enforces queue authorization and revocation. Verified by handler and race tests over mcpserver, dotbridge and dotevents (41.52s). --- internal/dotbridge/service.go | 5 ++- internal/mcpserver/events_discovery_test.go | 38 +++++++++++++++++++++ internal/mcpserver/events_gate_test.go | 22 ++++++++++-- 3 files changed, 61 insertions(+), 4 deletions(-) create mode 100644 internal/mcpserver/events_discovery_test.go diff --git a/internal/dotbridge/service.go b/internal/dotbridge/service.go index 39a27f7..65cd5ba 100644 --- a/internal/dotbridge/service.go +++ b/internal/dotbridge/service.go @@ -360,8 +360,11 @@ func (s *Service) MCPPrincipal() (string, string, error) { return s.mcpQueue, s.mcpQueue, nil } +// EventsList advertises the catalog whenever the bridge is enabled. Queue +// authorization is enforced on subscribe, not on discovery: the authenticated +// MCP consumer must be able to discover events before it can pick a queue. func (s *Service) EventsList(principal string) []dotevents.Definition { - if s.authorizeQueue(principal) != nil { + if s.gate() != nil { return []dotevents.Definition{} } return dotevents.Catalog() diff --git a/internal/mcpserver/events_discovery_test.go b/internal/mcpserver/events_discovery_test.go new file mode 100644 index 0000000..456a82e --- /dev/null +++ b/internal/mcpserver/events_discovery_test.go @@ -0,0 +1,38 @@ +package mcpserver + +import ( + "encoding/json" + "net/http/httptest" + "testing" +) + +// Discovery must work for the authenticated MCP consumer even when the caller's +// principal is not itself a paired client queue; authorization belongs to +// subscribe, which receives the queue in its arguments. +func TestEventsListDiscoverableBeforeSubscription(t *testing.T) { + s := openBridgeForMCPTest(t) + h, err := HTTPWithBridge(bridgeNoopBackend{}, s) + if err != nil { + t.Fatal(err) + } + server := httptest.NewServer(h) + defer server.Close() + env := eventsMCPRequest(t, server.URL, "events/list", eventsMeta()) + var listed struct { + Events []map[string]json.RawMessage `json:"events"` + } + if json.Unmarshal(env["result"], &listed) != nil || len(listed.Events) != 1 { + t.Fatalf("catalog must be discoverable before subscription: %s", env["result"]) + } + if string(listed.Events[0]["name"]) != `"dot.task.available"` { + t.Fatalf("unexpected event: %s", env["result"]) + } +} + +func eventsMeta() map[string]any { + return map[string]any{"_meta": map[string]any{ + "io.modelcontextprotocol/protocolVersion": "2026-07-28", + "io.modelcontextprotocol/clientCapabilities": map[string]any{}, + "io.modelcontextprotocol/clientInfo": map[string]any{"name": "fixture", "version": "1"}, + }} +} diff --git a/internal/mcpserver/events_gate_test.go b/internal/mcpserver/events_gate_test.go index c4c5ece..0d12fc5 100644 --- a/internal/mcpserver/events_gate_test.go +++ b/internal/mcpserver/events_gate_test.go @@ -4,6 +4,8 @@ import ( "encoding/json" "net/http/httptest" "testing" + + "github.com/dotcontrol/dotcontrol/internal/dotevents" ) // The events/list result keeps an array even when the bridge is disabled, and a @@ -46,12 +48,26 @@ func TestEventsListGateShapes(t *testing.T) { } } -func TestEventsListExcludesRevokedClientCatalog(t *testing.T) { +// A revoked client cannot subscribe, but the authenticated MCP consumer still +// discovers the catalog; authorization is enforced at subscribe time. +func TestEventsListStaysDiscoverableForRevokedClient(t *testing.T) { s := openBridgeForMCPTest(t) if err := s.RevokeClient("client.a"); err != nil { t.Fatal(err) } - if got := s.EventsList("client.a"); len(got) != 0 { - t.Fatalf("revoked client sees catalog: %#v", got) + if got := s.EventsList("client.a"); len(got) != 1 { + t.Fatalf("catalog must remain discoverable: %#v", got) + } + params := dotevents.Params{ + Name: dotevents.EventName, + Arguments: dotevents.Arguments{QueueID: "client.a"}, + Delivery: dotevents.Callback{ + Mode: "webhook", + URL: "https://callback.example/hook", + Secret: "whsec_MDEyMzQ1Njc4OWFiY2RlZjAxMjM0NTY3ODlhYmNkZWY=", + }, + } + if _, err := s.EventsSubscribe(t.Context(), "client.a", params); err == nil { + t.Fatal("revoked client must not subscribe") } } From 5b7d883495571f4dcb37e539560d5d460300d310 Mon Sep 17 00:00:00 2001 From: Pyuyi <204892921+VeraPyuyi@users.noreply.github.com> Date: Wed, 21 Oct 2026 00:16:26 +0800 Subject: [PATCH 37/47] fix(mcp-events): classify subscription errors and format event occurrence Map verification failures to JSON-RPC -32015 with classified data.reason (timeout, challenge_failed, url_rejected) so the platform can report specific causes. Align event occurrence body with the official eventId/name/timestamp/data schema. Verified with classified failure regression tests. --- internal/dotevents/delivery.go | 3 + internal/mcpserver/bridge.go | 48 +++++++++++- .../events_subscribe_failure_test.go | 78 +++++++++++++++++++ 3 files changed, 128 insertions(+), 1 deletion(-) create mode 100644 internal/mcpserver/events_subscribe_failure_test.go diff --git a/internal/dotevents/delivery.go b/internal/dotevents/delivery.go index e4460d3..0b11333 100644 --- a/internal/dotevents/delivery.go +++ b/internal/dotevents/delivery.go @@ -163,6 +163,9 @@ func (d *Delivery) deliverBytes(ctx context.Context, sub *subscriptionRecord, id } } func eventBodyBytes(n Notice) []byte { + // A top-level "type" identifies only protocol control notifications such as + // the verification challenge. An event occurrence carries eventId/name/ + // timestamp/data and a null cursor. return mustMarshal(struct { EventID string `json:"eventId"` Name string `json:"name"` diff --git a/internal/mcpserver/bridge.go b/internal/mcpserver/bridge.go index d055aed..82364d5 100644 --- a/internal/mcpserver/bridge.go +++ b/internal/mcpserver/bridge.go @@ -3,6 +3,7 @@ package mcpserver import ( "context" "encoding/json" + "errors" "fmt" "net/http" "time" @@ -38,6 +39,51 @@ type TaskStatusArgs struct { Token string `json:"token"` } +// callbackError is the MCP Events protocol failure shape: the specification +// requires JSON-RPC -32015 with a categorized data.reason so the platform can +// report why a subscription was refused. +type callbackError struct { + code int + reason string + msg string +} + +func (e *callbackError) Error() string { return e.msg } + +// callbackReason classifies a failed subscription verification. Unknown local +// failures are not mislabeled as endpoint errors; only transport, challenge and +// timeout categories are reported. +func callbackReason(err error) string { + if err == nil { + return "" + } + var ev *dotevents.Error + if !errors.As(err, &ev) { + return "server_error" + } + switch ev.Code { + case "network": + return "timeout" + case "verification": + return "challenge_failed" + case "callback": + return "url_rejected" + case "disabled", "capacity", "stale": + return "server_error" + default: + return "server_error" + } +} + +// eventsError converts a subscription failure into the MCP Events error shape. +func eventsError(err error) error { + if err == nil { + return nil + } + reason := callbackReason(err) + return &callbackError{code: -32015, reason: reason, msg: "dot_events_" + reason} +} + type EventsListParams struct { mcp.ParamsBase } @@ -131,7 +177,7 @@ func NewWithBridge(backend service.Backend, bridge *dotbridge.Service) (*mcp.Ser _ = queueID sub, err := bridge.EventsSubscribe(ctx, principal, dotevents.Params{Name: params.Name, Arguments: params.Arguments, Delivery: params.Delivery, Cursor: params.Cursor, TTL: params.TTL}) if err != nil { - return nil, err + return nil, eventsError(err) } return &EventsSubscribeResult{ID: sub.ID, RefreshBefore: sub.ExpiresAt, Cursor: nil, Truncated: false}, nil }); err != nil { diff --git a/internal/mcpserver/events_subscribe_failure_test.go b/internal/mcpserver/events_subscribe_failure_test.go new file mode 100644 index 0000000..28411b7 --- /dev/null +++ b/internal/mcpserver/events_subscribe_failure_test.go @@ -0,0 +1,78 @@ +package mcpserver + +import ( + "bytes" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "github.com/dotcontrol/dotcontrol/internal/dotevents" +) + +// rawMCPPost sends a JSON-RPC request without requiring a 200 result, so a +// protocol error body can be asserted. +func rawMCPPost(t *testing.T, url, method string, params any) (int, map[string]json.RawMessage) { + t.Helper() + payload, _ := json.Marshal(map[string]any{"jsonrpc": "2.0", "id": 1, "method": method, "params": params}) + req, _ := http.NewRequest(http.MethodPost, url, bytes.NewReader(payload)) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Accept", "application/json, text/event-stream") + req.Header.Set("Mcp-Method", method) + req.Header.Set("MCP-Protocol-Version", "2026-07-28") + res, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + defer res.Body.Close() + var env map[string]json.RawMessage + if err := json.NewDecoder(res.Body).Decode(&env); err != nil { + t.Fatalf("%s: undecodable body: %v", method, err) + } + return res.StatusCode, env +} + +// A failed callback verification must surface the documented -32015 code with a +// classified data.reason instead of an opaque error. +func TestEventsSubscribeCallbackFailureIsClassified(t *testing.T) { + s := openBridgeForMCPTest(t) + // No fixture transport: the default public transport rejects this callback. + s.ReplaceEventsDeliveryForTest(dotevents.NewDelivery(nil)) + h, err := HTTPWithBridge(bridgeNoopBackend{}, s) + if err != nil { + t.Fatal(err) + } + server := httptest.NewServer(h) + defer server.Close() + _, env := rawMCPPost(t, server.URL, "events/subscribe", map[string]any{ + "_meta": eventsMeta()["_meta"], + "name": "dot.task.available", + "arguments": map[string]string{"queue_id": "client.a"}, + "delivery": map[string]string{ + "mode": "webhook", + "url": "https://127.0.0.1.1/hook", + "secret": "whsec_MDEyMzQ1Njc4OWFiY2RlZjAxMjM0NTY3ODlhYmNkZWY=", + }, + "cursor": nil, + }) + raw, ok := env["error"] + if !ok { + t.Fatalf("expected a JSON-RPC error, got %s", env["result"]) + } + var e struct { + Code int `json:"code"` + Data json.RawMessage `json:"data"` + } + if json.Unmarshal(raw, &e) != nil { + t.Fatalf("undecodable error: %s", raw) + } + if e.Code != -32015 { + t.Fatalf("code = %d, want -32015: %s", e.Code, raw) + } + var data struct { + Reason string `json:"reason"` + } + if json.Unmarshal(e.Data, &data) != nil || data.Reason == "" { + t.Fatalf("missing classified reason: %s", raw) + } +} From ba58a24388ade52a49feacc4f55c7a76324e711b Mon Sep 17 00:00:00 2001 From: Pyuyi <204892921+VeraPyuyi@users.noreply.github.com> Date: Wed, 21 Oct 2026 01:17:54 +0800 Subject: [PATCH 38/47] fix(events): expose SDK errors and private subscription diagnostics Use jsonrpc.Error so callback failures carry real -32015 codes and data.reason. Distinguish network, timeout and URL policy rejection; retain allowlisted encrypted queue-scoped diagnostics across restart and suppress them after authorization changes. Expose only a stable bridge open failure code. Regression tests exercised prior failures; full repository race run passed in 68.12s, vet and diff checks passed. Actual platform subscription root cause remains unverified until deployed diagnostics observe an attempt. --- cmd/dotcontrol/runtime.go | 18 ++++++- internal/dotbridge/service.go | 8 ++++ .../subscribe_diagnostic_gate_test.go | 47 +++++++++++++++++++ internal/dotbridge/types.go | 29 ++++++++---- internal/dotevents/delivery.go | 10 ++++ internal/dotevents/manager.go | 47 +++++++++++++++++++ internal/dotevents/state.go | 17 +++++++ .../dotevents/subscribe_diagnostics_test.go | 44 +++++++++++++++++ internal/dotevents/transport_failure_test.go | 28 +++++++++++ internal/dotevents/types.go | 11 +++++ internal/dotevents/url_diagnostics_test.go | 36 ++++++++++++++ internal/dotevents/validate.go | 12 ++--- .../dotevents/validation_revision_test.go | 6 ++- internal/mcpserver/bridge.go | 46 ++++++------------ .../events_subscribe_failure_test.go | 2 +- 15 files changed, 309 insertions(+), 52 deletions(-) create mode 100644 internal/dotbridge/subscribe_diagnostic_gate_test.go create mode 100644 internal/dotevents/subscribe_diagnostics_test.go create mode 100644 internal/dotevents/transport_failure_test.go create mode 100644 internal/dotevents/url_diagnostics_test.go diff --git a/cmd/dotcontrol/runtime.go b/cmd/dotcontrol/runtime.go index 7390336..35acd95 100644 --- a/cmd/dotcontrol/runtime.go +++ b/cmd/dotcontrol/runtime.go @@ -56,6 +56,7 @@ type runtimeOwner struct { eng *engine.Engine bridge *codexbridge.Bridge dotBridge *dotbridge.Service + dotBridgeError string monitor *activity.Managed native *driver.Cua sup *tunnel.Supervisor @@ -182,7 +183,15 @@ func serve(ctx context.Context, dir string, c config.Config, v vault.Vault, args } owner := &runtimeOwner{ctx: ctx, stop: cancel, dir: dir, cfg: c, vault: v, eng: eng, monitor: monitor, sup: &tunnel.Supervisor{}, revoked: revErr == nil, noTunnel: *noTunnel, repair: make(chan struct{}, 1)} if c.DotBridgeEnabled { - owner.dotBridge, _ = openRuntimeDotBridge(ctx, dir, c, v, func() bool { return eng.Device().Paused }, func() bool { owner.mu.Lock(); defer owner.mu.Unlock(); return owner.revoked }) + // A bridge that fails to open must be visible: silently falling back to + // the 13-tool catalog would leave events/task tools missing with no + // operator-visible cause. + opened, openErr := openRuntimeDotBridge(ctx, dir, c, v, func() bool { return eng.Device().Paused }, func() bool { owner.mu.Lock(); defer owner.mu.Unlock(); return owner.revoked }) + if openErr != nil { + owner.dotBridgeError = "bridge_open_failed" + } else { + owner.dotBridge = opened + } } if owner.revoked { _ = eng.Pause(true) @@ -302,7 +311,12 @@ func serve(ctx context.Context, dir string, c config.Config, v vault.Vault, args } case "/admin/bridge/status": if owner.dotBridge == nil { - e = errors.New("bridge_unavailable") + // Surface the open failure instead of reporting a generic code. + if owner.dotBridgeError != "" { + out = map[string]any{"enabled": false, "available": false, "error": owner.dotBridgeError} + } else { + e = errors.New("bridge_unavailable") + } } else { out, e = owner.dotBridge.AdminStatus() } diff --git a/internal/dotbridge/service.go b/internal/dotbridge/service.go index 65cd5ba..887361e 100644 --- a/internal/dotbridge/service.go +++ b/internal/dotbridge/service.go @@ -260,6 +260,14 @@ func (s *Service) statusDTO(policy ClientPolicy) StatusDTO { out := StatusDTO{Version: APIVersion, Enabled: s.state.Enabled, Paired: exists && !current.Revoked, Code: "probe_required", Subscription: SubscriptionDTO{State: "inactive"}} if active { out.Subscription = SubscriptionDTO{State: "active", ExpiresAt: &sub.Expires} + } else if eventStatus.SubscribeFailure != nil && gateOK && exists && !current.Revoked && current.TokenHash == policy.TokenHash && s.capacityAllowedLocked() { + // A refused subscription is not silently "inactive": expose why the + // callback verification failed so an operator can fix the cause. + out.SubscribeFailure = &SubscribeFailureDTO{ + At: eventStatus.SubscribeFailure.At, + Reason: eventStatus.SubscribeFailure.Reason, + Stage: eventStatus.SubscribeFailure.Stage, + } } if gateOK && exists && !current.Revoked && current.TokenHash == policy.TokenHash && s.capacityAllowedLocked() && epoch == s.capacityEpoch && capErr == nil { out.Capacity = capacity diff --git a/internal/dotbridge/subscribe_diagnostic_gate_test.go b/internal/dotbridge/subscribe_diagnostic_gate_test.go new file mode 100644 index 0000000..984f1f1 --- /dev/null +++ b/internal/dotbridge/subscribe_diagnostic_gate_test.go @@ -0,0 +1,47 @@ +package dotbridge + +import ( + "github.com/dotcontrol/dotcontrol/internal/dotevents" + "testing" +) + +type diagnosticEvents struct { + EventsAPI + after func() +} + +func (e diagnosticEvents) Status(string, string) (dotevents.Status, error) { + if e.after != nil { + e.after() + } + return dotevents.Status{SubscribeFailure: &dotevents.SubscribeFailure{At: "2026-10-21T00:00:00Z", Reason: "verification", Stage: "challenge"}}, nil +} +func TestSubscribeDiagnosticFinalAuthorization(t *testing.T) { + for _, change := range []string{"none", "disable", "revoke"} { + t.Run(change, func(t *testing.T) { + s, _, id := openTestBridge(t) + s.mu.Lock() + delete(s.subscriptions, id) + s.mu.Unlock() + s.events = diagnosticEvents{EventsAPI: s.events, after: func() { + switch change { + case "disable": + if err := s.Disable(); err != nil { + t.Fatal(err) + } + case "revoke": + if err := s.RevokeClient(id); err != nil { + t.Fatal(err) + } + } + }} + st, err := s.StatusForClient(id) + if err != nil { + t.Fatal(err) + } + if (st.SubscribeFailure != nil) != (change == "none") { + t.Fatalf("diagnostic authorization: %#v", st) + } + }) + } +} diff --git a/internal/dotbridge/types.go b/internal/dotbridge/types.go index 91bf246..e59023a 100644 --- a/internal/dotbridge/types.go +++ b/internal/dotbridge/types.go @@ -73,16 +73,17 @@ type CapacityDTO struct { } type StatusDTO struct { - Version int `json:"version"` - Enabled bool `json:"enabled"` - Paired bool `json:"paired"` - Ready bool `json:"ready"` - Code string `json:"code"` - VerifiedAt *time.Time `json:"verified_at,omitempty"` - ExpiresAt *time.Time `json:"expires_at,omitempty"` - Subscription SubscriptionDTO `json:"subscription"` - Capabilities CapPolicy `json:"capabilities"` - Capacity *CapacityDTO `json:"capacity,omitempty"` + Version int `json:"version"` + Enabled bool `json:"enabled"` + Paired bool `json:"paired"` + Ready bool `json:"ready"` + Code string `json:"code"` + VerifiedAt *time.Time `json:"verified_at,omitempty"` + ExpiresAt *time.Time `json:"expires_at,omitempty"` + Subscription SubscriptionDTO `json:"subscription"` + Capabilities CapPolicy `json:"capabilities"` + Capacity *CapacityDTO `json:"capacity,omitempty"` + SubscribeFailure *SubscribeFailureDTO `json:"subscribe_failure,omitempty"` } type SubscriptionDTO struct { @@ -90,6 +91,14 @@ type SubscriptionDTO struct { ExpiresAt *time.Time `json:"expires_at,omitempty"` } +// SubscribeFailureDTO reports the most recent refused subscription attempt +// without exposing secrets or the callback URL. +type SubscribeFailureDTO struct { + At string `json:"at"` + Reason string `json:"reason"` + Stage string `json:"stage"` +} + type JobDTO struct { Version int `json:"version"` JobID string `json:"job_id"` diff --git a/internal/dotevents/delivery.go b/internal/dotevents/delivery.go index 0b11333..383c3aa 100644 --- a/internal/dotevents/delivery.go +++ b/internal/dotevents/delivery.go @@ -10,6 +10,7 @@ import ( "encoding/base64" "encoding/hex" "encoding/json" + "errors" "io" "net" "net/http" @@ -101,7 +102,16 @@ func (d *Delivery) post(ctx context.Context, sub *subscriptionRecord, id string, setWebhookHeaders(req, id, stamp, signPayload(sub, id, body, stamp), sub.ID) resp, e := d.httpClient().Do(req) if e != nil { + var networkError net.Error + timedOut := errors.Is(e, context.DeadlineExceeded) || errors.Is(ctx.Err(), context.DeadlineExceeded) || (errors.As(e, &networkError) && networkError.Timeout()) cancel() + var callbackErr *Error + if errors.As(e, &callbackErr) && callbackErr.Code == "callback" { + return nil, errf("callback") + } + if timedOut { + return nil, errf("timeout") + } return nil, errf("network") } resp.Body = &cancelBody{ReadCloser: resp.Body, cancel: cancel} diff --git a/internal/dotevents/manager.go b/internal/dotevents/manager.go index 672f7c6..ea3be5e 100644 --- a/internal/dotevents/manager.go +++ b/internal/dotevents/manager.go @@ -46,6 +46,17 @@ func (m *Manager) Subscribe(ctx context.Context, principal string, p Params) (Su func (m *Manager) subscribe(ctx context.Context, principal string, p Params, clock func() time.Time) (Subscription, error) { if e := validateParams(principal, p); e != nil { + if asEventsError(e).Code == "callback" { + if saveErr := m.store.withLock(false, true, func(st *persistenceState) error { + if !st.Enabled { + return errf("disabled") + } + st.LastSubscribe = &subscribeFailure{At: clock().UTC(), Principal: principal, Reason: "callback", Stage: "callback_url"} + return nil + }); saveErr != nil { + return Subscription{}, saveErr + } + } return Subscription{}, e } ttl, _ := parseTTL(p.TTL) @@ -107,6 +118,10 @@ func (m *Manager) subscribe(ctx context.Context, principal string, p Params, clo // Restore a previously accepted refresh only if no revoke/refresh won // the generation race. Never extend its TTL or cache after failure. restoreErr := m.store.withLock(false, true, func(st *persistenceState) error { + // Persist bounded, secret-free diagnostics before any rollback so an + // operator can see why the platform was refused. + code := asEventsError(e).Code + st.LastSubscribe = &subscribeFailure{At: clock().UTC(), Principal: principal, Reason: diagnosticReason(code), Stage: verifyStage(code)} current := st.Subscriptions[id] if !st.Enabled { return errf("disabled") @@ -148,6 +163,31 @@ func (m *Manager) subscribe(ctx context.Context, principal string, p Params, clo return Subscription{ID: id, ExpiresAt: rec.ExpiresAt, CallbackVerified: true}, nil } +// diagnosticReason accepts only stable categories, never arbitrary error text. +func diagnosticReason(code string) string { + switch code { + case "callback", "network", "timeout", "verification": + return code + default: + return "unknown" + } +} + +// verifyStage maps an internal callback error code to the phase that failed: +// URL validation, DNS/network reachability, or the challenge exchange itself. +func verifyStage(code string) string { + switch code { + case "callback": + return "callback_url" + case "network", "timeout": + return "network" + case "verification": + return "challenge" + default: + return "unknown" + } +} + // Never extend an accepted verification proof on refresh, even after a // backward clock step. Legacy deadlines need no guessed lock/IO allowance: // each reuse is capped by both subscription grants and the current cache TTL. @@ -358,6 +398,13 @@ func (m *Manager) Status(principal, q string) (Status, error) { out.Acknowledged = r.Acknowledged out.LastCode = r.LastCode } + if f := st.LastSubscribe; f != nil && f.Principal == principal && !out.CallbackVerified { + out.SubscribeFailure = &SubscribeFailure{ + At: f.At.UTC().Format(time.RFC3339), + Reason: f.Reason, + Stage: f.Stage, + } + } return nil }) return out, e diff --git a/internal/dotevents/state.go b/internal/dotevents/state.go index d765e9d..8e9e18b 100644 --- a/internal/dotevents/state.go +++ b/internal/dotevents/state.go @@ -56,6 +56,18 @@ type persistenceState struct { Subscriptions map[string]*subscriptionRecord `json:"subscriptions"` Deliveries map[string]*deliveryRecord `json:"deliveries"` QueueStatus map[string]*queueStatus `json:"queue_status"` + // LastSubscribe records the most recent verification failure so operators + // can see why a subscription was refused after it was discarded. + LastSubscribe *subscribeFailure `json:"last_subscribe,omitempty"` +} + +// subscribeFailure is bounded, secret-free diagnostics for the last failed +// events/subscribe attempt. +type subscribeFailure struct { + At time.Time `json:"at"` + Principal string `json:"principal"` + Reason string `json:"reason"` + Stage string `json:"stage"` } func newPersistenceState() *persistenceState { @@ -79,6 +91,11 @@ func parsePersistenceState(data []byte) (*persistenceState, error) { if st.StateVersion != stateVersion || st.Subscriptions == nil || st.Deliveries == nil || st.QueueStatus == nil || len(st.Subscriptions) > maxActiveSubs || len(st.Deliveries) > maxDeliveryRecords || len(st.QueueStatus) > maxDeliveryRecords { return nil, errf("corrupt") } + if f := st.LastSubscribe; f != nil { + if f.At.IsZero() || f.At.Year() < 1 || f.At.Year() > 9999 || validateID(f.Principal) != nil || diagnosticReason(f.Reason) != f.Reason || verifyStage(f.Reason) != f.Stage { + return nil, errf("corrupt") + } + } for id, sub := range st.Subscriptions { if sub != nil && !sub.VerifiedUntil.IsZero() { // Validate the persisted shape, not elapsed wall time. Legacy diff --git a/internal/dotevents/subscribe_diagnostics_test.go b/internal/dotevents/subscribe_diagnostics_test.go new file mode 100644 index 0000000..8062308 --- /dev/null +++ b/internal/dotevents/subscribe_diagnostics_test.go @@ -0,0 +1,44 @@ +package dotevents + +import ( + "context" + "encoding/json" + "net/http" + "strings" + "testing" +) + +func TestSubscribeFailureDiagnosticSurvivesReopenAndIsQueueScoped(t *testing.T) { + m, dir, closeFixture, _ := withFixtureServerURL(t, func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(200) + _, _ = w.Write([]byte(`{"challenge":"wrong"}`)) + }) + defer closeFixture() + if _, err := m.Subscribe(context.Background(), "alice", testParams("alice")); err == nil { + t.Fatal("wrong challenge accepted") + } + reopened, err := Open(dir, testKey, nil) + if err != nil { + t.Fatal(err) + } + status, err := reopened.Status("alice", "alice") + if err != nil { + t.Fatal(err) + } + if status.SubscribeFailure == nil || status.SubscribeFailure.Reason != "verification" || status.SubscribeFailure.Stage != "challenge" { + t.Fatalf("missing failure: %#v", status) + } + other, err := reopened.Status("bob", "bob") + if err != nil { + t.Fatal(err) + } + if other.SubscribeFailure != nil { + t.Fatal("another queue received failure diagnostic") + } + raw, _ := json.Marshal(status) + for _, secret := range []string{testSecret(), "callback.example", "wrong"} { + if strings.Contains(string(raw), secret) { + t.Fatalf("private data in diagnostic: %s", raw) + } + } +} diff --git a/internal/dotevents/transport_failure_test.go b/internal/dotevents/transport_failure_test.go new file mode 100644 index 0000000..6270cb2 --- /dev/null +++ b/internal/dotevents/transport_failure_test.go @@ -0,0 +1,28 @@ +package dotevents + +import ( + "context" + "errors" + "net/http" + "testing" +) + +type failingCallbackTransport struct{ err error } + +func (f failingCallbackTransport) RoundTrip(*http.Request) (*http.Response, error) { return nil, f.err } +func TestCallbackDeadlineIsDistinctFromNetworkFailure(t *testing.T) { + for _, tc := range []struct { + name string + err error + want string + }{{"timeout", context.DeadlineExceeded, "timeout"}, {"network", errors.New("private address or token must not escape"), "network"}} { + t.Run(tc.name, func(t *testing.T) { + d := NewDelivery(&http.Client{Transport: failingCallbackTransport{tc.err}}) + p := testParams("alice") + err := d.verify(context.Background(), &subscriptionRecord{ID: "sub.test", Params: p, Secret: p.Delivery.Secret}) + if got := asEventsError(err).Code; got != tc.want { + t.Fatalf("code %q want %q", got, tc.want) + } + }) + } +} diff --git a/internal/dotevents/types.go b/internal/dotevents/types.go index 511fd4c..c0c6dc1 100644 --- a/internal/dotevents/types.go +++ b/internal/dotevents/types.go @@ -54,4 +54,15 @@ type Status struct { Attempts int `json:"attempts"` Acknowledged bool `json:"acknowledged"` LastCode string `json:"last_code,omitempty"` + // SubscribeFailure explains the most recent refused subscription once it + // is no longer active. It carries no secret or callback URL. + SubscribeFailure *SubscribeFailure `json:"subscribe_failure,omitempty"` +} + +// SubscribeFailure is bounded, secret-free diagnostics for the last refused +// events/subscribe attempt. +type SubscribeFailure struct { + At string `json:"at"` + Reason string `json:"reason"` + Stage string `json:"stage"` } diff --git a/internal/dotevents/url_diagnostics_test.go b/internal/dotevents/url_diagnostics_test.go new file mode 100644 index 0000000..4f4b42a --- /dev/null +++ b/internal/dotevents/url_diagnostics_test.go @@ -0,0 +1,36 @@ +package dotevents + +import ( + "context" + "net/http" + "testing" +) + +func TestRejectedCallbackPersistsCategorizedDiagnostic(t *testing.T) { + for _, url := range []string{"http://example.com/hook", "https://localhost/hook", "https://127.0.0.1/hook"} { + t.Run(url, func(t *testing.T) { + m, _ := openTestManager(t) + p := testParams("alice") + p.Delivery.URL = url + _, err := m.Subscribe(context.Background(), "alice", p) + if got := asEventsError(err).Code; got != "callback" { + t.Fatalf("URL code %q, want callback", got) + } + st, err := m.Status("alice", "alice") + if err != nil { + t.Fatal(err) + } + if st.SubscribeFailure == nil || st.SubscribeFailure.Reason != "callback" || st.SubscribeFailure.Stage != "callback_url" { + t.Fatalf("missing URL diagnostic: %#v", st) + } + }) + } +} +func TestCallbackTransportPreservesAddressRejection(t *testing.T) { + d := NewDelivery(&http.Client{Transport: failingCallbackTransport{&Error{Code: "callback"}}}) + p := testParams("alice") + err := d.verify(context.Background(), &subscriptionRecord{ID: "sub.test", Params: p, Secret: p.Delivery.Secret}) + if got := asEventsError(err).Code; got != "callback" { + t.Fatalf("address rejection flattened to %q", got) + } +} diff --git a/internal/dotevents/validate.go b/internal/dotevents/validate.go index 1a26b5e..57be019 100644 --- a/internal/dotevents/validate.go +++ b/internal/dotevents/validate.go @@ -117,28 +117,28 @@ func parseTTL(raw json.RawMessage) (int64, error) { func validateCallbackURL(raw string) error { if len(raw) > maxCallbackURLBytes || !utf8.ValidString(raw) || strings.ContainsAny(raw, "\\\\#") { - return errf("unknown") + return errf("callback") } u, err := url.Parse(raw) if err != nil || u.Scheme != "https" || u.User != nil || u.Fragment != "" || u.Opaque != "" || u.Host == "" { - return errf("unknown") + return errf("callback") } host := strings.ToLower(strings.TrimSuffix(u.Hostname(), ".")) if host == "localhost" || strings.HasSuffix(host, ".localhost") || strings.HasSuffix(host, ".local") { - return errf("unknown") + return errf("callback") } if ip, err := netip.ParseAddr(host); err == nil { if !publicIP(ip) { - return errf("unknown") + return errf("callback") } return nil } if !hostPattern.MatchString(host) || !strings.Contains(host, ".") || strings.Contains(host, "..") { - return errf("unknown") + return errf("callback") } for _, label := range strings.Split(host, ".") { if len(label) > 63 || strings.HasPrefix(label, "-") || strings.HasSuffix(label, "-") { - return errf("unknown") + return errf("callback") } } return nil diff --git a/internal/dotevents/validation_revision_test.go b/internal/dotevents/validation_revision_test.go index 6bb2d77..788923d 100644 --- a/internal/dotevents/validation_revision_test.go +++ b/internal/dotevents/validation_revision_test.go @@ -33,7 +33,11 @@ func TestCompleteFixtureValidationZeroIO(t *testing.T) { p := testParams("alice") c.mutate(&p) _, e := m.Subscribe(context.Background(), c.principal, p) - if e == nil || asEventsError(e).Code != "unknown" { + want := "unknown" + if c.name == "private" || c.name == "fragment" { + want = "callback" + } + if e == nil || asEventsError(e).Code != want { t.Fatalf("validation error %v", e) } if calls.Load() != 0 { diff --git a/internal/mcpserver/bridge.go b/internal/mcpserver/bridge.go index 82364d5..4413546 100644 --- a/internal/mcpserver/bridge.go +++ b/internal/mcpserver/bridge.go @@ -12,6 +12,7 @@ import ( "github.com/dotcontrol/dotcontrol/internal/dotevents" "github.com/dotcontrol/dotcontrol/internal/dotqueue" "github.com/dotcontrol/dotcontrol/internal/service" + "github.com/modelcontextprotocol/go-sdk/jsonrpc" "github.com/modelcontextprotocol/go-sdk/mcp" ) @@ -39,49 +40,30 @@ type TaskStatusArgs struct { Token string `json:"token"` } -// callbackError is the MCP Events protocol failure shape: the specification -// requires JSON-RPC -32015 with a categorized data.reason so the platform can -// report why a subscription was refused. -type callbackError struct { - code int - reason string - msg string -} - -func (e *callbackError) Error() string { return e.msg } - -// callbackReason classifies a failed subscription verification. Unknown local -// failures are not mislabeled as endpoint errors; only transport, challenge and -// timeout categories are reported. -func callbackReason(err error) string { +// eventsError preserves the SDK's structured JSON-RPC error contract. +func eventsError(err error) error { if err == nil { - return "" + return nil } var ev *dotevents.Error if !errors.As(err, &ev) { - return "server_error" + return &jsonrpc.Error{Code: -32603, Message: "Subscription unavailable"} } + reason := "" switch ev.Code { + case "timeout": + reason = "timeout" case "network": - return "timeout" + reason = "network_error" case "verification": - return "challenge_failed" + reason = "challenge_failed" case "callback": - return "url_rejected" - case "disabled", "capacity", "stale": - return "server_error" + reason = "url_rejected" default: - return "server_error" - } -} - -// eventsError converts a subscription failure into the MCP Events error shape. -func eventsError(err error) error { - if err == nil { - return nil + return &jsonrpc.Error{Code: -32603, Message: "Subscription unavailable"} } - reason := callbackReason(err) - return &callbackError{code: -32015, reason: reason, msg: "dot_events_" + reason} + data, _ := json.Marshal(map[string]string{"reason": reason}) + return &jsonrpc.Error{Code: -32015, Message: "CallbackEndpointError", Data: data} } type EventsListParams struct { diff --git a/internal/mcpserver/events_subscribe_failure_test.go b/internal/mcpserver/events_subscribe_failure_test.go index 28411b7..65c46fa 100644 --- a/internal/mcpserver/events_subscribe_failure_test.go +++ b/internal/mcpserver/events_subscribe_failure_test.go @@ -50,7 +50,7 @@ func TestEventsSubscribeCallbackFailureIsClassified(t *testing.T) { "arguments": map[string]string{"queue_id": "client.a"}, "delivery": map[string]string{ "mode": "webhook", - "url": "https://127.0.0.1.1/hook", + "url": "https://127.0.0.1/hook", "secret": "whsec_MDEyMzQ1Njc4OWFiY2RlZjAxMjM0NTY3ODlhYmNkZWY=", }, "cursor": nil, From c7055e4d7b321cb5999977d9d4ba42a56e5e5e76 Mon Sep 17 00:00:00 2001 From: Pyuyi <204892921+VeraPyuyi@users.noreply.github.com> Date: Wed, 21 Oct 2026 01:29:07 +0800 Subject: [PATCH 39/47] fix(events): identify callback URL rejection rules safely Persist only allowlisted URL rejection rule names in queue-scoped diagnostics, preserving existing URL/public-address policy and legacy records. Subscription/status regression tests and related package race checks passed (41.81s); scoped source review approved. Actual callback rejection rule remains pending a new attempt. --- internal/dotbridge/service.go | 1 + internal/dotbridge/types.go | 1 + internal/dotevents/manager.go | 3 +- internal/dotevents/state.go | 3 +- internal/dotevents/types.go | 1 + internal/dotevents/url_rules_test.go | 27 ++++++++++++++++++ internal/dotevents/validate.go | 42 ++++++++++++++++++++++------ 7 files changed, 67 insertions(+), 11 deletions(-) create mode 100644 internal/dotevents/url_rules_test.go diff --git a/internal/dotbridge/service.go b/internal/dotbridge/service.go index 887361e..d6c0f77 100644 --- a/internal/dotbridge/service.go +++ b/internal/dotbridge/service.go @@ -267,6 +267,7 @@ func (s *Service) statusDTO(policy ClientPolicy) StatusDTO { At: eventStatus.SubscribeFailure.At, Reason: eventStatus.SubscribeFailure.Reason, Stage: eventStatus.SubscribeFailure.Stage, + Rule: eventStatus.SubscribeFailure.Rule, } } if gateOK && exists && !current.Revoked && current.TokenHash == policy.TokenHash && s.capacityAllowedLocked() && epoch == s.capacityEpoch && capErr == nil { diff --git a/internal/dotbridge/types.go b/internal/dotbridge/types.go index e59023a..33a8765 100644 --- a/internal/dotbridge/types.go +++ b/internal/dotbridge/types.go @@ -97,6 +97,7 @@ type SubscribeFailureDTO struct { At string `json:"at"` Reason string `json:"reason"` Stage string `json:"stage"` + Rule string `json:"rule,omitempty"` } type JobDTO struct { diff --git a/internal/dotevents/manager.go b/internal/dotevents/manager.go index ea3be5e..1b8948a 100644 --- a/internal/dotevents/manager.go +++ b/internal/dotevents/manager.go @@ -51,7 +51,7 @@ func (m *Manager) subscribe(ctx context.Context, principal string, p Params, clo if !st.Enabled { return errf("disabled") } - st.LastSubscribe = &subscribeFailure{At: clock().UTC(), Principal: principal, Reason: "callback", Stage: "callback_url"} + st.LastSubscribe = &subscribeFailure{At: clock().UTC(), Principal: principal, Reason: "callback", Stage: "callback_url", Rule: callbackURLRule(p.Delivery.URL)} return nil }); saveErr != nil { return Subscription{}, saveErr @@ -403,6 +403,7 @@ func (m *Manager) Status(principal, q string) (Status, error) { At: f.At.UTC().Format(time.RFC3339), Reason: f.Reason, Stage: f.Stage, + Rule: f.Rule, } } return nil diff --git a/internal/dotevents/state.go b/internal/dotevents/state.go index 8e9e18b..eda2a85 100644 --- a/internal/dotevents/state.go +++ b/internal/dotevents/state.go @@ -68,6 +68,7 @@ type subscribeFailure struct { Principal string `json:"principal"` Reason string `json:"reason"` Stage string `json:"stage"` + Rule string `json:"rule,omitempty"` } func newPersistenceState() *persistenceState { @@ -92,7 +93,7 @@ func parsePersistenceState(data []byte) (*persistenceState, error) { return nil, errf("corrupt") } if f := st.LastSubscribe; f != nil { - if f.At.IsZero() || f.At.Year() < 1 || f.At.Year() > 9999 || validateID(f.Principal) != nil || diagnosticReason(f.Reason) != f.Reason || verifyStage(f.Reason) != f.Stage { + if f.At.IsZero() || f.At.Year() < 1 || f.At.Year() > 9999 || validateID(f.Principal) != nil || diagnosticReason(f.Reason) != f.Reason || verifyStage(f.Reason) != f.Stage || !validCallbackRule(f.Rule) { return nil, errf("corrupt") } } diff --git a/internal/dotevents/types.go b/internal/dotevents/types.go index c0c6dc1..3f507de 100644 --- a/internal/dotevents/types.go +++ b/internal/dotevents/types.go @@ -65,4 +65,5 @@ type SubscribeFailure struct { At string `json:"at"` Reason string `json:"reason"` Stage string `json:"stage"` + Rule string `json:"rule,omitempty"` } diff --git a/internal/dotevents/url_rules_test.go b/internal/dotevents/url_rules_test.go new file mode 100644 index 0000000..7664a23 --- /dev/null +++ b/internal/dotevents/url_rules_test.go @@ -0,0 +1,27 @@ +package dotevents + +import ( + "context" + "strings" + "testing" +) + +func TestCallbackURLRuleDiagnostic(t *testing.T) { + for _, tc := range []struct{ url, rule string }{{"https://example.com/" + strings.Repeat("a", 2048), "url_length"}, {"http://example.com/hook", "https_required"}, {"https://localhost/hook", "local_hostname"}, {"https://127.0.0.1/hook", "nonpublic_address"}} { + t.Run(tc.rule, func(t *testing.T) { + m, _ := openTestManager(t) + p := testParams("alice") + p.Delivery.URL = tc.url + if _, err := m.Subscribe(context.Background(), "alice", p); err == nil { + t.Fatal("invalid URL accepted") + } + st, err := m.Status("alice", "alice") + if err != nil { + t.Fatal(err) + } + if st.SubscribeFailure == nil || st.SubscribeFailure.Rule != tc.rule { + t.Fatalf("diagnostic %#v want rule %s", st.SubscribeFailure, tc.rule) + } + }) + } +} diff --git a/internal/dotevents/validate.go b/internal/dotevents/validate.go index 57be019..6156148 100644 --- a/internal/dotevents/validate.go +++ b/internal/dotevents/validate.go @@ -116,32 +116,56 @@ func parseTTL(raw json.RawMessage) (int64, error) { } func validateCallbackURL(raw string) error { - if len(raw) > maxCallbackURLBytes || !utf8.ValidString(raw) || strings.ContainsAny(raw, "\\\\#") { + if callbackURLRule(raw) != "" { return errf("callback") } + return nil +} + +// callbackURLRule returns only a fixed policy category, never URL content. +func callbackURLRule(raw string) string { + if len(raw) > maxCallbackURLBytes { + return "url_length" + } + if !utf8.ValidString(raw) || strings.ContainsAny(raw, "\\#") { + return "url_characters" + } u, err := url.Parse(raw) - if err != nil || u.Scheme != "https" || u.User != nil || u.Fragment != "" || u.Opaque != "" || u.Host == "" { - return errf("callback") + if err != nil { + return "url_parse" + } + if u.Scheme != "https" { + return "https_required" + } + if u.User != nil || u.Fragment != "" || u.Opaque != "" || u.Host == "" { + return "url_structure" } host := strings.ToLower(strings.TrimSuffix(u.Hostname(), ".")) if host == "localhost" || strings.HasSuffix(host, ".localhost") || strings.HasSuffix(host, ".local") { - return errf("callback") + return "local_hostname" } if ip, err := netip.ParseAddr(host); err == nil { if !publicIP(ip) { - return errf("callback") + return "nonpublic_address" } - return nil + return "" } if !hostPattern.MatchString(host) || !strings.Contains(host, ".") || strings.Contains(host, "..") { - return errf("callback") + return "hostname_structure" } for _, label := range strings.Split(host, ".") { if len(label) > 63 || strings.HasPrefix(label, "-") || strings.HasSuffix(label, "-") { - return errf("callback") + return "hostname_label" } } - return nil + return "" +} +func validCallbackRule(rule string) bool { + switch rule { + case "", "url_length", "url_characters", "url_parse", "https_required", "url_structure", "local_hostname", "nonpublic_address", "hostname_structure", "hostname_label": + return true + } + return false } func publicIP(ip netip.Addr) bool { From 69ca10750307c3242b7abbfe2ef3a351b542807e Mon Sep 17 00:00:00 2001 From: Pyuyi <204892921+VeraPyuyi@users.noreply.github.com> Date: Wed, 21 Oct 2026 03:16:24 +0800 Subject: [PATCH 40/47] fix(events): resolve callback hosts over DoH for fake-ip networks Networks running fake-ip DNS (Clash, sing-box, Surge) answer every name with a synthetic address, so the public-address guard rejected the callback before any connection was attempted. Production diagnostics confirmed the path: the diagnostic carried reason=callback, stage=callback_url, rule="". The empty rule proved the rejection came from the dial path rather than URL pre-validation, and the encrypted state held zero subscription records. Resolve callback hostnames over DNS-over-HTTPS instead. The SSRF guard is unchanged: every candidate must still pass publicAddrs, the CNAME chain must begin at the queried host so a provider cannot substitute an unrelated hostname, and the system resolver remains the fallback. Implemented with the standard library only, because the Go module proxy is unreachable from this host. Classify failures honestly. A hijacked resolver is a network condition, so it now reports network rather than url_rejected, which had been sending operators to look for a URL problem that did not exist. publicTransport preserves the resolver's own error code instead of flattening it to callback. Providers are queried concurrently under one shared budget, and verified answers are memoized on the Delivery so a burst of deliveries and retries shares one resolution. Verified live on this host: chatgpt.com, openai.com and multi-hop CNAME hosts now resolve to public addresses. Full package race run passed in 51.55s. The cmd/dotcontrol legacy-reference failure is pre-existing on clean HEAD c7055e4 and unrelated. --- internal/dotevents/delivery.go | 29 +- internal/dotevents/doh.go | 398 +++++++++++++++++ internal/dotevents/doh_resolver_test.go | 559 ++++++++++++++++++++++++ 3 files changed, 984 insertions(+), 2 deletions(-) create mode 100644 internal/dotevents/doh.go create mode 100644 internal/dotevents/doh_resolver_test.go diff --git a/internal/dotevents/delivery.go b/internal/dotevents/delivery.go index 383c3aa..bfff663 100644 --- a/internal/dotevents/delivery.go +++ b/internal/dotevents/delivery.go @@ -17,6 +17,7 @@ import ( "net/netip" "strconv" "strings" + "sync" "time" ) @@ -31,7 +32,13 @@ var retrySchedule = []time.Duration{time.Second, 2 * time.Second, 4 * time.Secon // Delivery's zero value uses the public HTTPS transport. Test fixtures may // supply a custom HTTP client through NewDelivery. -type Delivery struct{ client *http.Client } +type Delivery struct { + client *http.Client + // resolver is built lazily for the production transport and then reused, so + // verified answers are memoized across deliveries and retries. + resolver *dohResolver + resolverOnce sync.Once +} // NewDelivery constructs a delivery transport with an explicit HTTP client. // Production callers should pass nil to use the public HTTPS transport. @@ -46,7 +53,17 @@ func (d *Delivery) httpClient() *http.Client { if d != nil && d.client != nil { return d.client } - return &http.Client{Timeout: maxHTTPTimeout, Transport: publicTransport(net.DefaultResolver.LookupIPAddr, (&net.Dialer{Timeout: maxHTTPTimeout}).DialContext), CheckRedirect: noRedirect} + // Resolve callback hostnames over DoH first. Networks running fake-ip DNS + // (Clash, sing-box, Surge) answer every name with a synthetic address that + // the public-address guard must reject, which would make every subscription + // fail before a connection was attempted. The system resolver remains the + // fallback, and the guard itself is unchanged. + // + // The resolver is built once and reused: it memoizes verified answers, so a + // burst of deliveries and retries shares one resolution instead of repeating + // the provider handshakes. + d.resolverOnce.Do(func() { d.resolver = newDoHResolver() }) + return &http.Client{Timeout: maxHTTPTimeout, Transport: publicTransport(d.resolver.lookup, (&net.Dialer{Timeout: maxHTTPTimeout}).DialContext), CheckRedirect: noRedirect} } func noRedirect(*http.Request, []*http.Request) error { return http.ErrUseLastResponse } @@ -65,6 +82,14 @@ func publicTransport(resolve resolveFunc, dial dialFunc) *http.Transport { } else { addrs, e = resolve(ctx, host) if e != nil { + // Preserve the resolver's own classification. A network or + // timeout failure must not be reported as a URL policy + // rejection, which would send the operator looking for a + // nonexistent configuration problem. + var eventsErr *Error + if errors.As(e, &eventsErr) { + return nil, eventsErr + } return nil, errf("callback") } } diff --git a/internal/dotevents/doh.go b/internal/dotevents/doh.go new file mode 100644 index 0000000..3c4c634 --- /dev/null +++ b/internal/dotevents/doh.go @@ -0,0 +1,398 @@ +package dotevents + +import ( + "context" + "encoding/json" + "io" + "net" + "net/http" + "net/netip" + "net/url" + "strings" + "sync" + "time" +) + +// Public DoH providers, queried concurrently. Order does not matter: every +// provider is asked under one shared deadline. No single provider is trusted — +// a hostile or poisoned answer must not be able to redirect a webhook, so a +// provider's answer is only accepted when it passes the public-address guard and +// the CNAME chain begins at the queried host. +var defaultDoHProviders = []string{ + "https://doh.pub/dns-query", + "https://cloudflare-dns.com/dns-query", + "https://dns.google/dns-query", +} + +const ( + // dohBudget is the total time allowed for all DoH queries combined. It is a + // fraction of the delivery deadline so that TLS and the verification + // challenge still have time to run. A single provider may never exceed it. + dohBudget = 4 * time.Second + // dohMaxBody bounds the response read so a hostile provider cannot exhaust + // memory with an oversized payload. + dohMaxBody = 64 * 1024 +) + +// dohResolver resolves callback hostnames over DNS-over-HTTPS. It exists +// because some networks hijack the system resolver (fake-ip mode in Clash, +// sing-box, Surge and similar), which returns a synthetic address for every +// name. That synthetic address fails the public-address guard, so every +// subscription would be rejected before a connection was attempted. +// +// The resolver never weakens that guard. It only changes how a real address is +// discovered: every candidate must pass publicAddrs, and the returned record +// name must match the queried host. The system resolver remains the fallback, +// and its answer is filtered the same way. +type dohResolver struct { + // providers is the list of DoH endpoints. Empty uses defaults. + providers []string + // budget bounds all DoH queries combined. Zero uses dohBudget. + budget time.Duration + // client performs the DoH requests. nil builds a proxy-bypassing client. + client *http.Client + // fallback resolves through the system resolver when DoH yields nothing. + // Injected in tests; nil means net.DefaultResolver. + fallback resolveFunc + + // mu guards cache. Delivery is called concurrently for many subscriptions, + // and each event retries, so answers are memoized briefly to avoid three + // new TLS handshakes per attempt. + mu sync.Mutex + cache map[string]cachedAnswer +} + +// cachedAnswer holds a verified answer for one host. Only addresses that passed +// publicAddrs are ever cached, so a cached entry cannot reintroduce a private +// address. The short TTL bounds staleness. +type cachedAnswer struct { + addrs []net.IPAddr + expiry time.Time +} + +// dohCacheTTL bounds how long a verified answer is reused. It is deliberately +// short: enough to absorb a burst of deliveries and retries, short enough that a +// legitimate address change is picked up quickly. +const dohCacheTTL = 60 * time.Second + +func newDoHResolver() *dohResolver { + return &dohResolver{providers: defaultDoHProviders, budget: dohBudget, cache: map[string]cachedAnswer{}} +} + +func (r *dohResolver) httpClient() *http.Client { + if r.client != nil { + return r.client + } + return &http.Client{ + Timeout: r.budgetOr(), + // DoH must bypass any configured proxy: the proxy may itself be the + // component rewriting DNS answers. + Transport: &http.Transport{Proxy: nil, DisableKeepAlives: true}, + } +} + +func (r *dohResolver) budgetOr() time.Duration { + if r.budget > 0 { + return r.budget + } + return dohBudget +} + +func (r *dohResolver) fallbackResolver() resolveFunc { + if r.fallback != nil { + return r.fallback + } + return net.DefaultResolver.LookupIPAddr +} + +func (r *dohResolver) providerList() []string { + if len(r.providers) == 0 { + return defaultDoHProviders + } + return r.providers +} + +// lookup returns public addresses for host. +// +// Every provider is queried concurrently under one shared deadline so a slow or +// blackholed provider cannot consume the delivery budget. A provider's answer +// counts only when the record name matches the queried host and every address +// passes publicAddrs, so neither a fake-ip answer nor a hijacked answer for a +// different hostname can be used. +// +// Errors are classified for the operator: a URL that the policy guard rejects +// is "callback"; everything else (no provider reachable, no usable answer, or a +// system resolver that only yields non-public addresses) is "network", which is +// what it actually is. It is never reported as a URL policy rejection. +func (r *dohResolver) lookup(ctx context.Context, host string) ([]net.IPAddr, error) { + host = strings.ToLower(strings.TrimSuffix(host, ".")) + if host == "" || len(host) > 253 { + return nil, errf("callback") + } + // Literal addresses need no resolution, but the guard still applies. The + // caller re-validates, and rejecting here keeps the classification honest. + if ip, err := netip.ParseAddr(host); err == nil { + if !publicAddrs([]net.IPAddr{{IP: ip.AsSlice(), Zone: ip.Zone()}}) { + return nil, errf("callback") + } + return []net.IPAddr{{IP: ip.AsSlice(), Zone: ip.Zone()}}, nil + } + if cached, ok := r.cached(host); ok { + return cached, nil + } + addrs, err := r.resolve(ctx, host) + if err != nil { + return nil, err + } + r.remember(host, addrs) + return addrs, nil +} + +// resolve performs the actual provider queries and fallback. It is separated +// from lookup so only verified, public answers are memoized. +// +// The error code describes the true cause so an operator is not sent looking for +// a configuration problem that does not exist: +// +// - "network" no provider could be reached, or the system resolver failed. +// This is the fake-ip case: DNS is answering, but not usefully. +// - "callback" the URL itself violates policy (a non-public literal host), +// or every resolver returned an address the guard must reject. +// +// A hijacked resolver is a network condition, not a URL policy rejection, so it +// is reported as "network" and the operator is pointed at DNS. +func (r *dohResolver) resolve(ctx context.Context, host string) ([]net.IPAddr, error) { + if out := r.queryProviders(ctx, host); len(out) > 0 { + return out, nil + } + // DoH produced nothing usable. Fall back to the system resolver so behavior + // is never worse than before, and let the guard reject it if the answer is + // still non-public. + addrs, e := r.fallbackResolver()(ctx, host) + if e != nil || len(addrs) == 0 { + return nil, errf("network") + } + if !publicAddrs(addrs) { + // The system resolver returned only addresses the guard must reject. + // That is a hijacked or fake-ip resolver, which is a network condition. + return nil, errf("network") + } + return addrs, nil +} + +func (r *dohResolver) cached(host string) ([]net.IPAddr, bool) { + r.mu.Lock() + defer r.mu.Unlock() + if r.cache == nil { + return nil, false + } + e, ok := r.cache[host] + if !ok || !time.Now().Before(e.expiry) || len(e.addrs) == 0 { + return nil, false + } + // Return a copy: the dial path must not be able to mutate a shared slice. + out := make([]net.IPAddr, len(e.addrs)) + copy(out, e.addrs) + return out, true +} + +func (r *dohResolver) remember(host string, addrs []net.IPAddr) { + r.mu.Lock() + defer r.mu.Unlock() + if r.cache == nil { + r.cache = map[string]cachedAnswer{} + } + out := make([]net.IPAddr, len(addrs)) + copy(out, addrs) + r.cache[host] = cachedAnswer{addrs: out, expiry: time.Now().Add(dohCacheTTL)} +} + +// queryProviders asks every provider concurrently and returns the union of +// addresses that passed the guard. Providers are never retried, so the total +// work is bounded by the shared deadline. +func (r *dohResolver) queryProviders(ctx context.Context, host string) []net.IPAddr { + providers := r.providerList() + if len(providers) == 0 { + return nil + } + ctx, cancel := context.WithTimeout(ctx, r.budgetOr()) + defer cancel() + type result struct { + addrs []net.IPAddr + } + results := make(chan result, len(providers)) + var wg sync.WaitGroup + for _, provider := range providers { + wg.Add(1) + go func(provider string) { + defer wg.Done() + addrs, ok := r.queryProvider(ctx, provider, host) + if !ok { + return + } + results <- result{addrs: addrs} + }(provider) + } + wg.Wait() + close(results) + seen := map[string]bool{} + var out []net.IPAddr + for res := range results { + for _, a := range res.addrs { + // Defensive: the guard is applied here as well as per provider, so + // no address can reach the dial path without passing it. + if !publicAddrs([]net.IPAddr{a}) || seen[a.IP.String()] { + continue + } + seen[a.IP.String()] = true + out = append(out, a) + } + } + return out +} + +// queryProvider asks one DoH endpoint. The bool reports whether the provider +// gave a well-formed answer for this host that passed the public-address guard. +func (r *dohResolver) queryProvider(ctx context.Context, provider, host string) ([]net.IPAddr, bool) { + endpoint, err := url.Parse(provider) + if err != nil || endpoint.Scheme != "https" || endpoint.Host == "" { + return nil, false + } + q := endpoint.Query() + q.Set("name", host) + q.Set("type", "A") + endpoint.RawQuery = q.Encode() + req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint.String(), nil) + if err != nil { + return nil, false + } + req.Header.Set("Accept", "application/dns-json") + resp, err := r.httpClient().Do(req) + if err != nil { + return nil, false + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusOK { + return nil, false + } + body, err := io.ReadAll(io.LimitReader(resp.Body, dohMaxBody)) + if err != nil { + return nil, false + } + return parseDoHAnswer(body, host) +} + +// parseDoHAnswer extracts the A records that answer a query for host. +// +// A real answer section is the wire answer verbatim, so a CNAME'd host arrives +// as a type-5 record followed by A records owned by the CNAME target. Demanding +// an exact owner name would discard every such answer, silently reverting +// CDN-fronted hosts to the hijacked system resolver. The chain is therefore +// followed from the queried name, which also preserves the security property: an +// address is accepted only when it is reachable by following CNAME records that +// start at the queried host. A provider cannot substitute an unrelated hostname, +// because the chain must begin at host and each link must be present. +// +// SERVFAIL, malformed JSON, a broken chain, non-A records and unparseable or +// non-public addresses all yield "no answer". +func parseDoHAnswer(body []byte, host string) ([]net.IPAddr, bool) { + var payload struct { + Status int `json:"status"` + Answer []struct { + Name string `json:"name"` + Type int `json:"type"` + Data string `json:"data"` + } `json:"Answer"` + } + if err := json.Unmarshal(body, &payload); err != nil { + return nil, false + } + // NOERROR only. SERVFAIL, REFUSED and NXDOMAIN are not answers. + if payload.Status != 0 { + return nil, false + } + normalize := func(s string) string { return strings.ToLower(strings.TrimSuffix(strings.TrimSpace(s), ".")) } + want := normalize(host) + if want == "" { + return nil, false + } + // Collect the CNAME links and the addresses for each owner name. + owners := map[string]*cnameEntry{} + for _, a := range payload.Answer { + name := normalize(a.Name) + if name == "" { + continue + } + e, ok := owners[name] + if !ok { + e = &cnameEntry{} + owners[name] = e + } + switch a.Type { + case 1: + ip, err := netip.ParseAddr(strings.TrimSpace(a.Data)) + if err != nil || !ip.Unmap().Is4() { + continue + } + // Only public addresses may be returned. This is the same guard the + // dial path applies, so the two layers cannot disagree. + if !publicAddrs([]net.IPAddr{{IP: ip.AsSlice()}}) { + continue + } + e.a = append(e.a, ip) + case 5: + target := normalize(a.Data) + if target == "" || target == name { + continue + } + // Only the first link for an owner is honored; a second, different + // link is an attempt to fork the chain. + if e.cname != "" && e.cname != target { + continue + } + e.cname = target + } + } + // Walk the chain from the queried host, bounded by the number of records so + // a cycle cannot spin. + addrs := walkCNAMEChain(owners, want, len(payload.Answer)) + if len(addrs) == 0 { + return nil, false + } + out := make([]net.IPAddr, 0, len(addrs)) + for _, ip := range addrs { + out = append(out, net.IPAddr{IP: ip.AsSlice()}) + } + return out, true +} + +// cnameEntry collects the CNAME target and the public addresses for one owner +// name within a single DoH answer section. +type cnameEntry struct { + cname string + a []netip.Addr +} + +// walkCNAMEChain follows CNAME links from the queried host and returns the +// public addresses owned by the terminal target. Addresses at intermediate names +// are not used: only the final target resolves the query. +func walkCNAMEChain(owners map[string]*cnameEntry, start string, limit int) []netip.Addr { + seen := map[string]bool{} + var out []netip.Addr + for cur := start; cur != "" && limit > 0; limit-- { + if seen[cur] { + return nil + } + seen[cur] = true + e, ok := owners[cur] + if !ok { + return nil + } + out = append(out, e.a...) + if e.cname == "" { + return out + } + cur = e.cname + } + return nil +} diff --git a/internal/dotevents/doh_resolver_test.go b/internal/dotevents/doh_resolver_test.go new file mode 100644 index 0000000..e6f098a --- /dev/null +++ b/internal/dotevents/doh_resolver_test.go @@ -0,0 +1,559 @@ +package dotevents + +import ( + "context" + "crypto/tls" + "crypto/x509" + "encoding/json" + "net" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" +) + +// dohAnswer mirrors the subset of RFC 8484 JSON the resolver consumes. +type dohAnswer struct { + Name string `json:"name"` + Type int `json:"type"` + TTL int `json:"TTL"` + Data string `json:"data"` +} + +// dohResponse is the provider payload shape. +type dohResponse struct { + Status int `json:"status"` + Answer []dohAnswer `json:"Answer"` +} + +// dohFixture describes one provider's canned behavior. +type dohFixture struct { + // records are returned verbatim, letting a test set a foreign Name. + records []dohAnswer + status int + // raw, when set, is written instead of encoding records. + raw string +} + +// newDoHServer builds a fixture DoH endpoint over TLS (the production client +// only accepts https). The returned client trusts every fixture's certificate. +func newDoHServer(t *testing.T, fx dohFixture) (*httptest.Server, string) { + t.Helper() + s := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if fx.raw != "" { + w.Header().Set("Content-Type", "application/dns-json") + _, _ = w.Write([]byte(fx.raw)) + return + } + w.Header().Set("Content-Type", "application/dns-json") + _ = json.NewEncoder(w).Encode(dohResponse{Status: fx.status, Answer: fx.records}) + })) + return s, s.URL +} + +// aRecord builds a well-formed A record for the given host. +func aRecord(host, ip string) dohAnswer { + return dohAnswer{Name: strings.ToLower(strings.TrimSuffix(host, ".")) + ".", Type: 1, TTL: 60, Data: ip} +} + +// fixtureClient builds a client trusting the given fixtures. Empty is valid. +func fixtureClient(t *testing.T, servers ...*httptest.Server) *http.Client { + t.Helper() + pool := x509.NewCertPool() + for _, s := range servers { + pool.AddCert(s.Certificate()) + } + return &http.Client{Timeout: 3 * time.Second, Transport: &http.Transport{ + Proxy: nil, + DisableKeepAlives: true, + TLSClientConfig: &tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS12}, + }} +} + +// resolverFor builds a resolver over the given fixtures, with no fallback so a +// test cannot silently pass because the system resolver supplied an answer. +func resolverFor(t *testing.T, fixtures ...dohFixture) *dohResolver { + t.Helper() + servers := make([]*httptest.Server, 0, len(fixtures)) + urls := make([]string, 0, len(fixtures)) + for _, fx := range fixtures { + s, u := newDoHServer(t, fx) + servers = append(servers, s) + urls = append(urls, u) + } + t.Cleanup(func() { + for _, s := range servers { + s.Close() + } + }) + return &dohResolver{ + providers: urls, + budget: 2 * time.Second, + client: fixtureClient(t, servers...), + fallback: func(ctx context.Context, host string) ([]net.IPAddr, error) { return nil, context.DeadlineExceeded }, + } +} + +// TestDoHResolverAcceptsDistinctPublicAnswers verifies the realistic case: two +// providers each return a different but public address (normal for CDNs). Both +// must be accepted rather than demanding byte-identical answers. +func TestDoHResolverAcceptsDistinctPublicAnswers(t *testing.T) { + r := resolverFor(t, + dohFixture{records: []dohAnswer{aRecord("chatgpt.com", "104.18.32.47")}}, + dohFixture{records: []dohAnswer{aRecord("chatgpt.com", "172.64.155.209")}}, + ) + got, err := r.lookup(context.Background(), "chatgpt.com") + if err != nil { + t.Fatalf("lookup: %v", err) + } + if len(got) != 2 { + t.Fatalf("got %v, want both public answers", got) + } +} + +// TestDoHResolverIgnoresFakeIP verifies a provider answering with a fake-ip +// address is discarded, so the resolver can never smuggle 198.18.0.0/16 past the +// public-address guard. +func TestDoHResolverIgnoresFakeIP(t *testing.T) { + r := resolverFor(t, + dohFixture{records: []dohAnswer{aRecord("chatgpt.com", "198.18.0.21")}}, + dohFixture{records: []dohAnswer{aRecord("chatgpt.com", "104.18.32.47")}}, + ) + got, err := r.lookup(context.Background(), "chatgpt.com") + if err != nil { + t.Fatalf("lookup: %v", err) + } + if len(got) != 1 || got[0].String() != "104.18.32.47" { + t.Fatalf("got %v, want only the public answer", got) + } +} + +// TestDoHResolverRejectsEveryPrivateRange verifies no private, loopback, +// link-local, CGNAT, benchmark or special-use range can be returned by a +// provider. This is the core SSRF property. +func TestDoHResolverRejectsEveryPrivateRange(t *testing.T) { + for _, ip := range []string{ + "198.18.0.21", "198.51.100.7", "10.0.0.1", "127.0.0.1", "169.254.169.254", + "100.64.0.1", "192.0.2.1", "203.0.113.1", "192.168.1.1", "172.16.0.1", + "0.0.0.0", "224.0.0.1", "255.255.255.255", "::1", "fe80::1", "fc00::1", + "::ffff:198.18.0.21", "::ffff:127.0.0.1", + } { + t.Run(ip, func(t *testing.T) { + r := resolverFor(t, dohFixture{records: []dohAnswer{aRecord("chatgpt.com", ip)}}) + if got, err := r.lookup(context.Background(), "chatgpt.com"); err == nil { + t.Fatalf("provider answer %s returned %v, want rejection", ip, got) + } + }) + } +} + +// TestDoHResolverAllFakeIPFails verifies that when every provider returns a +// non-public answer the resolver fails rather than returning it. +func TestDoHResolverAllFakeIPFails(t *testing.T) { + r := resolverFor(t, + dohFixture{records: []dohAnswer{aRecord("chatgpt.com", "198.18.0.21")}}, + dohFixture{records: []dohAnswer{aRecord("chatgpt.com", "198.18.0.22")}}, + ) + if got, err := r.lookup(context.Background(), "chatgpt.com"); err == nil { + t.Fatalf("expected failure, got %v", got) + } +} + +// TestDoHResolverAllFakeIPIsNetworkNotCallback verifies the operator sees the +// truth: an all-fake-ip network condition must not be reported as a URL policy +// rejection. +func TestDoHResolverAllFakeIPIsNetworkNotCallback(t *testing.T) { + r := resolverFor(t, + dohFixture{records: []dohAnswer{aRecord("chatgpt.com", "198.18.0.21")}}, + dohFixture{records: []dohAnswer{aRecord("chatgpt.com", "198.18.0.22")}}, + ) + _, err := r.lookup(context.Background(), "chatgpt.com") + if got := asEventsError(err).Code; got != "network" { + t.Fatalf("code = %q, want network (not callback/url_rejected)", got) + } +} + +// TestDoHResolverRejectsForeignRecordName verifies a hostile provider cannot +// answer a query for the callback host with records for a different hostname, +// which would redirect the webhook and its secret to an attacker-chosen host. +func TestDoHResolverRejectsForeignRecordName(t *testing.T) { + r := resolverFor(t, dohFixture{records: []dohAnswer{ + {Name: "evil.example.", Type: 1, TTL: 60, Data: "104.18.32.47"}, + }}) + if got, err := r.lookup(context.Background(), "chatgpt.com"); err == nil { + t.Fatalf("foreign record name accepted: %v", got) + } +} + +// system resolver is used, so behavior is never worse than before the change. +func TestDoHResolverFallsBackToSystem(t *testing.T) { + r := &dohResolver{ + providers: []string{"https://127.0.0.1:1/unreachable"}, + budget: 500 * time.Millisecond, + client: fixtureClient(t), + fallback: func(ctx context.Context, host string) ([]net.IPAddr, error) { + return []net.IPAddr{{IP: net.ParseIP("104.18.32.47")}}, nil + }, + } + got, err := r.lookup(context.Background(), "chatgpt.com") + if err != nil { + t.Fatalf("lookup: %v", err) + } + if len(got) != 1 || got[0].String() != "104.18.32.47" { + t.Fatalf("got %v, want the fallback answer", got) + } +} + +// TestDoHResolverFallbackNonPublicIsRejected verifies the fallback path cannot +// bypass the public-address guard when the system resolver is hijacked, and that +// it is reported as a network condition rather than a URL policy rejection. +func TestDoHResolverFallbackNonPublicIsRejected(t *testing.T) { + r := &dohResolver{ + providers: []string{"https://127.0.0.1:1/unreachable"}, + budget: 500 * time.Millisecond, + client: fixtureClient(t), + fallback: func(ctx context.Context, host string) ([]net.IPAddr, error) { + return []net.IPAddr{{IP: net.ParseIP("198.18.0.21")}}, nil + }, + } + _, err := r.lookup(context.Background(), "chatgpt.com") + if got := asEventsError(err).Code; got != "network" { + t.Fatalf("code = %q, want network (a hijacked resolver is not a URL problem)", got) + } +} + +// TestDoHResolverFallbackMixedAnswersAreRejected verifies the all-or-nothing +// semantics: a fallback slice containing any non-public address is rejected +// whole, so one poisoned answer cannot slip through alongside a good one. +func TestDoHResolverFallbackMixedAnswersAreRejected(t *testing.T) { + r := &dohResolver{ + providers: []string{"https://127.0.0.1:1/unreachable"}, + budget: 500 * time.Millisecond, + client: fixtureClient(t), + fallback: func(ctx context.Context, host string) ([]net.IPAddr, error) { + return []net.IPAddr{{IP: net.ParseIP("104.18.32.47")}, {IP: net.ParseIP("127.0.0.1")}}, nil + }, + } + if got, err := r.lookup(context.Background(), "chatgpt.com"); err == nil { + t.Fatalf("mixed fallback accepted: %v", got) + } +} + +// TestDoHResolverRejectsBadStatusAndOversized verifies provider failures are +// treated as "no answer" rather than poisoning the result. +func TestDoHResolverRejectsBadStatusAndOversized(t *testing.T) { + r := resolverFor(t, + dohFixture{status: 2}, + dohFixture{raw: `{"Status":0,"Answer":[{"name":"chatgpt.com.","type":1,"TTL":60,"data":"` + strings.Repeat("1", 4096) + `"}]}`}, + ) + if got, err := r.lookup(context.Background(), "chatgpt.com"); err == nil { + t.Fatalf("expected failure, got %v", got) + } +} + +// TestDoHResolverRejectsNonHTTPSProvider verifies a plaintext or malformed +// provider URL is skipped rather than trusted. +func TestDoHResolverRejectsNonHTTPSProvider(t *testing.T) { + for _, provider := range []string{"http://doh.example/dns-query", "not a url", "ftp://doh.example/dns-query", ""} { + r := &dohResolver{ + providers: []string{provider}, + budget: 500 * time.Millisecond, + client: fixtureClient(t), + fallback: func(ctx context.Context, host string) ([]net.IPAddr, error) { return nil, context.DeadlineExceeded }, + } + if got, err := r.lookup(context.Background(), "chatgpt.com"); err == nil { + t.Fatalf("provider %q accepted: %v", provider, got) + } + } +} + +// TestDoHResolverRejectsOversizedHost verifies the host length bound. +func TestDoHResolverRejectsOversizedHost(t *testing.T) { + r := resolverFor(t, dohFixture{records: []dohAnswer{aRecord("chatgpt.com", "104.18.32.47")}}) + long := strings.Repeat("a.", 200) + "com" + if got, err := r.lookup(context.Background(), long); err == nil { + t.Fatalf("oversized host accepted: %v", got) + } +} + +// TestDoHResolverLiteralIPIsGuarded verifies the literal-address short path is +// itself policed, so it cannot be used to reach a private address. +func TestDoHResolverLiteralIPIsGuarded(t *testing.T) { + r := resolverFor(t, dohFixture{records: []dohAnswer{aRecord("chatgpt.com", "104.18.32.47")}}) + for _, host := range []string{"127.0.0.1", "198.18.0.21", "10.0.0.1", "169.254.169.254", "::1"} { + if got, err := r.lookup(context.Background(), host); err == nil { + t.Fatalf("literal %q accepted: %v", host, got) + } + } + // A public literal is returned unchanged. + got, err := r.lookup(context.Background(), "104.18.32.47") + if err != nil || len(got) != 1 || got[0].String() != "104.18.32.47" { + t.Fatalf("public literal: got %v err %v", got, err) + } +} + +// TestDoHResolverBudgetIsShared verifies a single slow provider cannot consume +// the whole delivery budget: queries run concurrently under one deadline. +func TestDoHResolverBudgetIsShared(t *testing.T) { + release := make(chan struct{}) + slow := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + select { + case <-release: + case <-r.Context().Done(): + } + })) + defer slow.Close() + defer close(release) + fast, fastURL := newDoHServer(t, dohFixture{records: []dohAnswer{aRecord("chatgpt.com", "104.18.32.47")}}) + defer fast.Close() + + r := &dohResolver{ + providers: []string{slow.URL, fastURL}, + budget: 1 * time.Second, + client: fixtureClient(t, slow, fast), + fallback: func(ctx context.Context, host string) ([]net.IPAddr, error) { return nil, context.DeadlineExceeded }, + } + start := time.Now() + got, err := r.lookup(context.Background(), "chatgpt.com") + elapsed := time.Since(start) + if err != nil { + t.Fatalf("lookup: %v", err) + } + if len(got) != 1 || got[0].String() != "104.18.32.47" { + t.Fatalf("got %v", got) + } + if elapsed > 2*time.Second { + t.Fatalf("took %v, want the shared 1s budget to bound the slow provider", elapsed) + } +} + +// TestDoHResolverCachesVerifiedAnswers verifies a burst of deliveries reuses one +// resolution instead of repeating the provider queries and TLS handshakes. +func TestDoHResolverCachesVerifiedAnswers(t *testing.T) { + var queries atomic.Int64 + fixture := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + queries.Add(1) + w.Header().Set("Content-Type", "application/dns-json") + _ = json.NewEncoder(w).Encode(dohResponse{Answer: []dohAnswer{aRecord("chatgpt.com", "104.18.32.47")}}) + })) + defer fixture.Close() + + r := &dohResolver{ + providers: []string{fixture.URL}, + budget: 2 * time.Second, + client: fixtureClient(t, fixture), + fallback: func(ctx context.Context, host string) ([]net.IPAddr, error) { return nil, context.DeadlineExceeded }, + } + for i := 0; i < 5; i++ { + got, err := r.lookup(context.Background(), "chatgpt.com") + if err != nil || len(got) != 1 || got[0].String() != "104.18.32.47" { + t.Fatalf("lookup %d: got %v err %v", i, got, err) + } + } + if n := queries.Load(); n != 1 { + t.Fatalf("provider queried %d times, want 1 (cached)", n) + } +} + +// TestDoHResolverDoesNotCacheRejections verifies a rejected host is retried +// rather than memoized, so a transient network failure is not made permanent. +func TestDoHResolverDoesNotCacheRejections(t *testing.T) { + var queries atomic.Int64 + fixture := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + queries.Add(1) + w.Header().Set("Content-Type", "application/dns-json") + _ = json.NewEncoder(w).Encode(dohResponse{Answer: []dohAnswer{aRecord("chatgpt.com", "198.18.0.21")}}) + })) + defer fixture.Close() + + r := &dohResolver{ + providers: []string{fixture.URL}, + budget: 2 * time.Second, + client: fixtureClient(t, fixture), + fallback: func(ctx context.Context, host string) ([]net.IPAddr, error) { return nil, context.DeadlineExceeded }, + } + for i := 0; i < 3; i++ { + if _, err := r.lookup(context.Background(), "chatgpt.com"); err == nil { + t.Fatal("rejection was memoized and later returned an address") + } + } + if n := queries.Load(); n != 3 { + t.Fatalf("provider queried %d times, want 3 (no caching of failures)", n) + } +} + +// TestDoHResolverCacheReturnsCopy verifies the dial path cannot mutate a cached +// slice and corrupt later lookups. +func TestDoHResolverCacheReturnsCopy(t *testing.T) { + fixture, fixtureURL := newDoHServer(t, dohFixture{records: []dohAnswer{aRecord("chatgpt.com", "104.18.32.47"), aRecord("chatgpt.com", "172.64.155.209")}}) + defer fixture.Close() + r := &dohResolver{ + providers: []string{fixtureURL}, + budget: 2 * time.Second, + client: fixtureClient(t, fixture), + fallback: func(ctx context.Context, host string) ([]net.IPAddr, error) { return nil, context.DeadlineExceeded }, + } + first, err := r.lookup(context.Background(), "chatgpt.com") + if err != nil { + t.Fatal(err) + } + first[0] = net.IPAddr{IP: net.ParseIP("127.0.0.1")} + second, err := r.lookup(context.Background(), "chatgpt.com") + if err != nil { + t.Fatal(err) + } + if second[0].String() != "104.18.32.47" { + t.Fatalf("cached slice was mutated: %v", second) + } +} + +// TestCallbackDialUsesDoHBeforeFakeIP verifies the end-to-end dial path: with a +// hijacked system resolver, DoH supplies the public address so verification +// proceeds instead of being rejected at the dial stage. +func TestCallbackDialUsesDoHBeforeFakeIP(t *testing.T) { + doh, dohURL := newDoHServer(t, dohFixture{records: []dohAnswer{aRecord("chatgpt.com", "104.18.32.47")}}) + defer doh.Close() + + var dialed string + r := &dohResolver{ + providers: []string{dohURL}, + budget: 2 * time.Second, + client: fixtureClient(t, doh), + fallback: func(ctx context.Context, host string) ([]net.IPAddr, error) { + // Simulates the hijacked system resolver (fake-ip mode). + return []net.IPAddr{{IP: net.ParseIP("198.18.0.21")}}, nil + }, + } + d := NewDelivery(&http.Client{Transport: publicTransport(r.lookup, func(ctx context.Context, network, address string) (net.Conn, error) { + dialed = address + return nil, errf("dial_unreachable") + })}) + err := d.verify(context.Background(), &subscriptionRecord{ + ID: "sub.probe", + Params: Params{Delivery: Callback{URL: "https://chatgpt.com/hook", Secret: testSecret()}}, + Secret: testSecret(), + }) + if got := asEventsError(err).Code; got != "network" { + t.Fatalf("code = %q, want network (dial attempted against the DoH address)", got) + } + if dialed != "104.18.32.47:443" { + t.Fatalf("dialed %q, want the DoH-provided public address", dialed) + } +} + +// TestCallbackDialStillRejectsFakeIPAfterDoHFailure verifies that if DoH cannot +// help and the system resolver is hijacked, the guard still refuses the +// connection — the resolver must not have opened a bypass. +func TestCallbackDialStillRejectsFakeIPAfterDoHFailure(t *testing.T) { + var dialed string + r := &dohResolver{ + providers: []string{"https://127.0.0.1:1/unreachable"}, + budget: 500 * time.Millisecond, + client: fixtureClient(t), + fallback: func(ctx context.Context, host string) ([]net.IPAddr, error) { + return []net.IPAddr{{IP: net.ParseIP("198.18.0.21")}}, nil + }, + } + d := NewDelivery(&http.Client{Transport: publicTransport(r.lookup, func(ctx context.Context, network, address string) (net.Conn, error) { + dialed = address + return nil, errf("dial_unreachable") + })}) + err := d.verify(context.Background(), &subscriptionRecord{ + ID: "sub.probe", + Params: Params{Delivery: Callback{URL: "https://chatgpt.com/hook", Secret: testSecret()}}, + Secret: testSecret(), + }) + if got := asEventsError(err).Code; got != "network" { + t.Fatalf("code = %q, want network", got) + } + if dialed != "" { + t.Fatalf("dialed %q, want no dial attempt", dialed) + } +} + +// TestDoHResolverFollowsCNAMEChain verifies the realistic CDN answer shape: a +// type-5 record followed by A records owned by the CNAME target. The addresses +// behind the chain must be returned, not discarded. +func TestDoHResolverFollowsCNAMEChain(t *testing.T) { + r := resolverFor(t, dohFixture{records: []dohAnswer{ + {Name: "chatgpt.com.", Type: 5, TTL: 60, Data: "chatgpt.com.cdn.cloudflare.net."}, + {Name: "chatgpt.com.cdn.cloudflare.net.", Type: 1, TTL: 60, Data: "104.18.32.47"}, + {Name: "chatgpt.com.cdn.cloudflare.net.", Type: 1, TTL: 60, Data: "172.64.155.209"}, + }}) + got, err := r.lookup(context.Background(), "chatgpt.com") + if err != nil { + t.Fatalf("lookup: %v", err) + } + if len(got) != 2 { + t.Fatalf("got %v, want the addresses behind the CNAME chain", got) + } +} + +// TestDoHResolverFollowsDeepCNAMEChain verifies a multi-hop chain, as returned +// by real providers (e.g. a 4-hop chain for outlook.office.com). +func TestDoHResolverFollowsDeepCNAMEChain(t *testing.T) { + r := resolverFor(t, dohFixture{records: []dohAnswer{ + {Name: "outlook.office.com.", Type: 5, TTL: 60, Data: "outlook.cloud.microsoft."}, + {Name: "outlook.cloud.microsoft.", Type: 5, TTL: 60, Data: "acdcatm.outlook.mira.tm.svc.cloud.microsoft."}, + {Name: "acdcatm.outlook.mira.tm.svc.cloud.microsoft.", Type: 5, TTL: 60, Data: "shed.outlook.acdc.tm.svc.cloud.microsoft."}, + {Name: "shed.outlook.acdc.tm.svc.cloud.microsoft.", Type: 1, TTL: 60, Data: "40.104.72.18"}, + }}) + got, err := r.lookup(context.Background(), "outlook.office.com") + if err != nil { + t.Fatalf("lookup: %v", err) + } + if len(got) != 1 || got[0].String() != "40.104.72.18" { + t.Fatalf("got %v, want the terminal target address", got) + } +} + +// TestDoHResolverRejectsChainNotStartingAtHost verifies the security property +// that motivated name matching: a provider may not substitute an unrelated +// hostname, because the CNAME chain must begin at the queried host. +func TestDoHResolverRejectsChainNotStartingAtHost(t *testing.T) { + r := resolverFor(t, dohFixture{records: []dohAnswer{ + {Name: "evil.example.", Type: 5, TTL: 60, Data: "target.example."}, + {Name: "target.example.", Type: 1, TTL: 60, Data: "104.18.32.47"}, + }}) + if got, err := r.lookup(context.Background(), "chatgpt.com"); err == nil { + t.Fatalf("foreign chain accepted: %v", got) + } +} + +// TestDoHResolverRejectsBrokenCNAMEChain verifies a chain that does not terminate +// at an address is discarded rather than partially used. +func TestDoHResolverRejectsBrokenCNAMEChain(t *testing.T) { + r := resolverFor(t, dohFixture{records: []dohAnswer{ + {Name: "chatgpt.com.", Type: 5, TTL: 60, Data: "missing.example."}, + {Name: "unrelated.example.", Type: 1, TTL: 60, Data: "104.18.32.47"}, + }}) + if got, err := r.lookup(context.Background(), "chatgpt.com"); err == nil { + t.Fatalf("broken chain accepted: %v", got) + } +} + +// TestDoHResolverRejectsCNAMELoop verifies a cyclic chain cannot spin. +func TestDoHResolverRejectsCNAMELoop(t *testing.T) { + r := resolverFor(t, dohFixture{records: []dohAnswer{ + {Name: "chatgpt.com.", Type: 5, TTL: 60, Data: "a.example."}, + {Name: "a.example.", Type: 5, TTL: 60, Data: "b.example."}, + {Name: "b.example.", Type: 5, TTL: 60, Data: "chatgpt.com."}, + }}) + if got, err := r.lookup(context.Background(), "chatgpt.com"); err == nil { + t.Fatalf("CNAME loop accepted: %v", got) + } +} + +// TestDoHResolverCNAMEtoPrivateTargetIsRejected verifies a chain that leads to a +// non-public address is refused, so CNAME following cannot become an SSRF path. +func TestDoHResolverCNAMEtoPrivateTargetIsRejected(t *testing.T) { + r := resolverFor(t, dohFixture{records: []dohAnswer{ + {Name: "chatgpt.com.", Type: 5, TTL: 60, Data: "internal.example."}, + {Name: "internal.example.", Type: 1, TTL: 60, Data: "127.0.0.1"}, + }}) + if got, err := r.lookup(context.Background(), "chatgpt.com"); err == nil { + t.Fatalf("chain to a private target accepted: %v", got) + } +} From 019b4ee9db26d8a942aba0ef166652e606a3feea Mon Sep 17 00:00:00 2001 From: Pyuyi <204892921+VeraPyuyi@users.noreply.github.com> Date: Wed, 21 Oct 2026 11:05:27 +0800 Subject: [PATCH 41/47] fix(bridge): preserve verified capability across trusted subscription renewals Persist subscription trust epochs separately from verification attempts and retain mode proofs for continuous renewals of the same authenticated callback binding. Tie readiness to the committed active lease instead of requiring a new manual nonce every ten minutes. Changed callback, signing secret, authorization, unsubscribe or an expired lease breaks continuity; legacy proofs do not gain sustainable trust without binding evidence. Expose sanitized renewal timestamps and failures while retaining valid old grants. ChatGPT remains responsible for refreshing before refreshBefore; DotControl never renews its own authorization. Existing accepted-job queries remain independent from readiness. Sample grant time under the state lock and recheck cached finalization so lock waits cannot preserve an expired epoch. Honor the minimum committed bridge and events lease after partial persistence. Deterministic regressions failed before the fixes and passed afterward; full repository race tests passed in 73.73s. Independent review closed both lease-boundary findings with no new important findings. Live migration and renewal acceptance are pending joint deployment. --- internal/dotbridge/authorization.go | 34 +++++++ internal/dotbridge/gates_test.go | 5 +- internal/dotbridge/probe.go | 2 +- internal/dotbridge/service.go | 96 +++++++++++++++++-- .../sustainable_partial_commit_test.go | 67 +++++++++++++ internal/dotbridge/sustainable_replay_test.go | 21 ++++ internal/dotbridge/sustainable_test.go | 85 ++++++++++++++++ internal/dotbridge/task_mode_test.go | 3 +- internal/dotbridge/task_proof_test.go | 6 +- internal/dotbridge/types.go | 19 ++-- internal/dotbridge/worker.go | 5 +- internal/dotevents/manager.go | 53 ++++++++-- internal/dotevents/state.go | 2 + internal/dotevents/store.go | 5 + internal/dotevents/trust.go | 8 ++ internal/dotevents/trust_failure_test.go | 51 ++++++++++ internal/dotevents/trust_lock_test.go | 67 +++++++++++++ internal/dotevents/trust_test.go | 71 ++++++++++++++ internal/dotevents/types.go | 21 ++-- 19 files changed, 586 insertions(+), 35 deletions(-) create mode 100644 internal/dotbridge/sustainable_partial_commit_test.go create mode 100644 internal/dotbridge/sustainable_replay_test.go create mode 100644 internal/dotbridge/sustainable_test.go create mode 100644 internal/dotevents/trust.go create mode 100644 internal/dotevents/trust_failure_test.go create mode 100644 internal/dotevents/trust_lock_test.go create mode 100644 internal/dotevents/trust_test.go diff --git a/internal/dotbridge/authorization.go b/internal/dotbridge/authorization.go index 16eb51a..0520c01 100644 --- a/internal/dotbridge/authorization.go +++ b/internal/dotbridge/authorization.go @@ -5,9 +5,16 @@ import ( "encoding/hex" "errors" "time" + + "github.com/dotcontrol/dotcontrol/internal/dotevents" ) type subscriptionProof struct { + BindingID string + TrustEpoch string + AuthHash string + Consumer string + Refreshed time.Time Expires time.Time Generation uint64 } @@ -45,7 +52,34 @@ func (s *Service) binding(job, token string) (claimBinding, error) { func (s *Service) activeSubscription(queue string) (subscriptionProof, bool) { s.mu.Lock() sub, ok := s.subscriptions[queue] + policy := s.state.Clients[queue] + consumer := s.mcpQueue s.mu.Unlock() + if sub.TrustEpoch != "" { + if sub.AuthHash != policy.TokenHash || sub.Consumer != consumer { + return sub, false + } + if reader, supported := s.events.(interface { + TrustedBindings(string) ([]dotevents.TrustedBinding, error) + }); supported { + bindings, err := reader.TrustedBindings(queue) + if err != nil { + return sub, false + } + found := false + for _, b := range bindings { + if b.ID == sub.BindingID && b.Epoch == sub.TrustEpoch { + found = true + if b.Expires.Before(sub.Expires) { + sub.Expires = b.Expires + } + } + } + if !found { + return sub, false + } + } + } return sub, ok && time.Now().Before(sub.Expires) } func consumerWorker(consumer, worker string) string { diff --git a/internal/dotbridge/gates_test.go b/internal/dotbridge/gates_test.go index 1a71898..ff5ab4a 100644 --- a/internal/dotbridge/gates_test.go +++ b/internal/dotbridge/gates_test.go @@ -110,7 +110,10 @@ func TestReadinessSubscriptionGenerationAndExpiry(t *testing.T) { t.Fatal(e) } answer := c.Submission.Messages[0].Content[len("Reply with exactly this nonce: "):] - // Renewing a subscription invalidates claims made under its previous generation. + // Breaking the binding invalidates claims made under its previous generation. + if e = s.EventsUnsubscribe(context.Background(), id, testEventsParams(id)); e != nil { + t.Fatal(e) + } VerifyTestSubscription(t, s, id) if _, e = s.ClaimTask(id, p.JobID, "attempt"); e != nil { t.Fatal(e) diff --git a/internal/dotbridge/probe.go b/internal/dotbridge/probe.go index ad261ef..ca7d5b6 100644 --- a/internal/dotbridge/probe.go +++ b/internal/dotbridge/probe.go @@ -81,7 +81,7 @@ func (s *Service) StartProbeMode(clientID, mode string) (ProbeResult, error) { if !s.state.Enabled || !time.Now().Before(current.Expires) || current.Generation != subscription.Generation || !current.Expires.Equal(subscription.Expires) { return ProbeResult{}, errors.New("subscription_changed") } - policy.setProof(mode, ProbeProof{JobID: job.ID, AnswerHash: hashToken(answer), Generation: subscription.Generation, SubscriptionExpires: subscription.Expires}) + policy.setProof(mode, ProbeProof{TrustEpoch: subscription.TrustEpoch, JobID: job.ID, AnswerHash: hashToken(answer), Generation: subscription.Generation, SubscriptionExpires: subscription.Expires}) s.state.Clients[clientID] = policy if e := s.saveState(); e != nil { return ProbeResult{}, e diff --git a/internal/dotbridge/service.go b/internal/dotbridge/service.go index d6c0f77..5c5f0ae 100644 --- a/internal/dotbridge/service.go +++ b/internal/dotbridge/service.go @@ -257,12 +257,28 @@ func (s *Service) statusDTO(policy ClientPolicy) StatusDTO { sub := s.subscriptions[current.QueueID] now := time.Now() active := now.Before(sub.Expires) + if sub.TrustEpoch != "" { + active = false + for _, b := range eventStatus.Bindings { + if b.ID == sub.BindingID && b.Epoch == sub.TrustEpoch && sub.AuthHash == current.TokenHash && sub.Consumer == s.mcpQueue && now.Before(b.Expires) { + if b.Expires.Before(sub.Expires) { + sub.Expires = b.Expires + } + sub.Refreshed = b.Refreshed + // Both stores must authorize the effective lease. An event-only + // renewal cannot extend the bridge's last committed grant. + active = now.Before(sub.Expires) + } + } + } out := StatusDTO{Version: APIVersion, Enabled: s.state.Enabled, Paired: exists && !current.Revoked, Code: "probe_required", Subscription: SubscriptionDTO{State: "inactive"}} if active { - out.Subscription = SubscriptionDTO{State: "active", ExpiresAt: &sub.Expires} - } else if eventStatus.SubscribeFailure != nil && gateOK && exists && !current.Revoked && current.TokenHash == policy.TokenHash && s.capacityAllowedLocked() { + out.Subscription = SubscriptionDTO{State: "active", ExpiresAt: &sub.Expires, RefreshBefore: &sub.Expires, LastRefreshedAt: &sub.Refreshed, RenewalState: "current"} + } + if eventStatus.SubscribeFailure != nil && gateOK && exists && !current.Revoked && current.TokenHash == policy.TokenHash && s.capacityAllowedLocked() { // A refused subscription is not silently "inactive": expose why the // callback verification failed so an operator can fix the cause. + out.Subscription.RenewalState = "failed" out.SubscribeFailure = &SubscribeFailureDTO{ At: eventStatus.SubscribeFailure.At, Reason: eventStatus.SubscribeFailure.Reason, @@ -273,15 +289,32 @@ func (s *Service) statusDTO(policy ClientPolicy) StatusDTO { if gateOK && exists && !current.Revoked && current.TokenHash == policy.TokenHash && s.capacityAllowedLocked() && epoch == s.capacityEpoch && capErr == nil { out.Capacity = capacity } + if !sub.Expires.IsZero() && !active { + out.Code = "subscription_expired" + out.Subscription.State = "expired" + } + if !s.state.Enabled { + out.Code = "disabled" + } + if s.paused != nil && s.paused() { + out.Code = "bridge_paused" + } + if current.Revoked || (s.revoked != nil && s.revoked()) { + out.Code = "access_revoked" + } if !gateOK || !s.state.Enabled || !exists || current.Revoked || !active || e != nil || !eventStatus.Subscribed || !eventStatus.CallbackVerified || (s.paused != nil && s.paused()) || (s.revoked != nil && s.revoked()) { return out } for _, mode := range []string{dotqueue.ModeAnswer, dotqueue.ModeTask} { proof := current.proof(mode) - if proof.Verified.IsZero() || !proof.Consumed || proof.Generation != sub.Generation || !now.Before(proof.Expires) || !proof.SubscriptionExpires.Equal(sub.Expires) { + trusted := proof.TrustEpoch != "" && proof.TrustEpoch == sub.TrustEpoch + if proof.Verified.IsZero() || !proof.Consumed || proof.Generation != sub.Generation || ((proof.TrustEpoch != "" || sub.TrustEpoch != "") && !trusted) || (!trusted && (!now.Before(proof.Expires) || !proof.SubscriptionExpires.Equal(sub.Expires))) { continue } expiry := proof.Expires + if trusted { + expiry = sub.Expires + } if sub.Expires.Before(expiry) { expiry = sub.Expires } @@ -383,6 +416,9 @@ func (s *Service) EventsSubscribe(ctx context.Context, principal string, params if e := s.authorizeQueue(queue); e != nil { return dotevents.Subscription{}, e } + s.mu.Lock() + epoch := s.capacityEpoch + s.mu.Unlock() sub, e := s.events.Subscribe(ctx, queue, params) if e != nil { return sub, e @@ -396,15 +432,36 @@ func (s *Service) EventsSubscribe(ctx context.Context, principal string, params if p.Revoked { return dotevents.Subscription{}, errors.New("access_revoked") } + if !s.capacityAllowedLocked() || epoch != s.capacityEpoch { + return dotevents.Subscription{}, errors.New("bridge_disabled") + } + if reader, ok := s.events.(interface { + TrustedBindings(string) ([]dotevents.TrustedBinding, error) + }); ok { + bindings, readErr := reader.TrustedBindings(queue) + matched := false + for _, b := range bindings { + if b.ID == sub.ID && b.Epoch == sub.TrustEpoch && b.Expires.Equal(sub.ExpiresAt) { + matched = true + } + } + if readErr != nil || !matched { + return dotevents.Subscription{}, errors.New("subscription_changed") + } + } old := s.subscriptions[queue] - s.subscriptions[queue] = subscriptionProof{Expires: sub.ExpiresAt, Generation: old.Generation + 1} - // Keep consumed timestamps for audit/replay protection. Generation binding - // invalidates both modes without erasing their first verification time. - p.Capabilities = CapPolicy{} - p.ProbeExpires = time.Time{} - p.TaskProbe.Expires = time.Time{} + generation := old.Generation + 1 + if sub.TrustEpoch != "" && sub.TrustEpoch == old.TrustEpoch && old.BindingID == sub.ID && old.AuthHash == p.TokenHash && old.Consumer == s.mcpQueue { + generation = old.Generation + } + s.subscriptions[queue] = subscriptionProof{Expires: sub.ExpiresAt, Generation: generation, BindingID: sub.ID, TrustEpoch: sub.TrustEpoch, AuthHash: p.TokenHash, Consumer: s.mcpQueue, Refreshed: time.Now().UTC()} + if generation != old.Generation { + p.Capabilities = CapPolicy{} + p.ProbeExpires = time.Time{} + p.TaskProbe.Expires = time.Time{} + } s.state.Clients[queue] = p - return sub, nil + return sub, s.saveState() } func (s *Service) EventsUnsubscribe(ctx context.Context, principal string, params dotevents.Params) error { queue := params.Arguments.QueueID @@ -417,6 +474,9 @@ func (s *Service) EventsUnsubscribe(ctx context.Context, principal string, param old.Expires = time.Time{} old.Generation++ s.subscriptions[queue] = old + if saveErr := s.saveState(); e == nil { + e = saveErr + } s.mu.Unlock() if post := s.authorizeQueue(queue); post != nil { return post @@ -456,10 +516,26 @@ func (s *Service) loadState() error { st.Clients = map[string]ClientPolicy{} } s.state = st + if st.Version != APIVersion { + return errors.New("bridge_state_corrupt") + } + if st.Subscriptions != nil { + for queue, sub := range st.Subscriptions { + policy, ok := st.Clients[queue] + if !ok || policy.QueueID != queue || sub.Generation == 0 { + return errors.New("bridge_state_corrupt") + } + if sub.TrustEpoch != "" && (len(sub.BindingID) != 64 || sub.AuthHash == "" || sub.Consumer == "") { + return errors.New("bridge_state_corrupt") + } + } + s.subscriptions = st.Subscriptions + } return nil } func (s *Service) saveState() error { + s.state.Subscriptions = s.subscriptions data, err := json.MarshalIndent(s.state, "", " ") if err != nil { return err diff --git a/internal/dotbridge/sustainable_partial_commit_test.go b/internal/dotbridge/sustainable_partial_commit_test.go new file mode 100644 index 0000000..2bfab1c --- /dev/null +++ b/internal/dotbridge/sustainable_partial_commit_test.go @@ -0,0 +1,67 @@ +package dotbridge + +import ( + "context" + "testing" + "time" + + "github.com/dotcontrol/dotcontrol/internal/dotqueue" +) + +func TestRestoredExpiredBridgeLeaseCannotAdoptEventRenewal(t *testing.T) { + s, _, id := openTestBridge(t) + proveSustainable(t, s, id) + policy := s.state.Clients[id] + accepted, err := s.queue.Submit(id, dotqueue.Submission{RequestID: "accepted.before.expiry", Mode: dotqueue.ModeAnswer, SessionID: "recovery", Messages: []dotqueue.Message{{Role: "user", Content: "accepted"}}}) + if err != nil { + t.Fatal(err) + } + // Model event renewal committed with the same epoch, while the bridge still + // has its previous committed lease (now expired). + if _, err = s.events.Subscribe(context.Background(), id, testEventsParams(id)); err != nil { + t.Fatal(err) + } + s.mu.Lock() + sub := s.subscriptions[id] + sub.Expires = time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC) + s.subscriptions[id] = sub + err = s.saveState() + s.mu.Unlock() + if err != nil { + t.Fatal(err) + } + if err = s.Close(); err != nil { + t.Fatal(err) + } + restored, err := Open(context.Background(), Config{Root: s.root, Listen: s.listen, Vault: s.vault, MCPQueue: s.mcpQueue}) + if err != nil { + t.Fatal(err) + } + defer restored.Close() + status, err := restored.StatusForClient(id) + if err != nil { + t.Fatal(err) + } + if status.Ready || status.Code != "subscription_expired" { + t.Errorf("expired committed lease became ready: %#v", status) + } + before, err := restored.queue.List(id) + if err != nil { + t.Fatal(err) + } + _, _, err = restored.submit(policy, SubmitRequest{RequestID: "new.after.expiry", Mode: dotqueue.ModeAnswer, SessionID: "new", Messages: []MessageDTO{{Role: "user", Content: "must refuse"}}}) + if err == nil { + t.Error("new submission accepted expired bridge lease") + } + after, err := restored.queue.List(id) + if err != nil { + t.Fatal(err) + } + if len(before) != len(after) { + t.Error("unready submission created a job") + } + recovered, err := restored.getJob(policy, accepted.ID) + if err != nil || recovered.JobID != accepted.ID { + t.Fatalf("accepted job recovery: %#v %v", recovered, err) + } +} diff --git a/internal/dotbridge/sustainable_replay_test.go b/internal/dotbridge/sustainable_replay_test.go new file mode 100644 index 0000000..ef5ba92 --- /dev/null +++ b/internal/dotbridge/sustainable_replay_test.go @@ -0,0 +1,21 @@ +package dotbridge + +import ( + "testing" + "strings" + "github.com/dotcontrol/dotcontrol/internal/dotqueue" +) + +func TestSustainableConsumedProofReplayDoesNotRefreshEvidence(t *testing.T){ + s,_,id:=openTestBridge(t) + p,err:=s.StartProbe(id);if err!=nil{t.Fatal(err)} + c,err:=s.ClaimTask(id,p.JobID,"replay-continuous");if err!=nil{t.Fatal(err)} + result:=dotqueue.Result{Text:strings.TrimPrefix(c.Submission.Messages[0].Content,"Reply with exactly this nonce: ")} + if _,err=s.CompleteTask(p.JobID,c.Token,result);err!=nil{t.Fatal(err)} + s.mu.Lock();before:=s.state.Clients[id].proof(dotqueue.ModeAnswer);s.mu.Unlock() + VerifyTestSubscription(t,s,id) + if _,err=s.CompleteTask(p.JobID,c.Token,result);err!=nil{t.Fatal(err)} + s.mu.Lock();after:=s.state.Clients[id].proof(dotqueue.ModeAnswer);s.mu.Unlock() + if before!=after{t.Fatal("replayed completion altered consumed proof")} + status,_:=s.StatusForClient(id);if !status.Ready{t.Fatal("real renewal lost consumed capability")} +} diff --git a/internal/dotbridge/sustainable_test.go b/internal/dotbridge/sustainable_test.go new file mode 100644 index 0000000..db29a86 --- /dev/null +++ b/internal/dotbridge/sustainable_test.go @@ -0,0 +1,85 @@ +package dotbridge + +import ( + "context" + "strings" + "testing" + "time" + + "github.com/dotcontrol/dotcontrol/internal/dotqueue" +) + +func proveSustainable(t *testing.T, s *Service, id string) { + t.Helper() + p, err := s.StartProbe(id) + if err != nil { + t.Fatal(err) + } + c, err := s.ClaimTask(id, p.JobID, "sustainable") + if err != nil { + t.Fatal(err) + } + nonce := strings.TrimPrefix(c.Submission.Messages[0].Content, "Reply with exactly this nonce: ") + if _, err = s.CompleteTask(p.JobID, c.Token, dotqueue.Result{Text: nonce}); err != nil { + t.Fatal(err) + } +} + +func TestSustainableRenewalAndRestart(t *testing.T) { + s, _, id := openTestBridge(t) + proveSustainable(t, s, id) + s.mu.Lock() + p := s.state.Clients[id] + proof := p.proof(dotqueue.ModeAnswer) + // A mode proven more than ten minutes ago remains valid through its grant. + proof.Verified = time.Now().Add(-11 * time.Minute) + p.setProof(dotqueue.ModeAnswer, proof) + s.state.Clients[id] = p + s.mu.Unlock() + VerifyTestSubscription(t, s, id) + status, _ := s.StatusForClient(id) + if !status.Ready { + t.Fatalf("continuous renewal lost proof: %#v", status) + } + if err := s.Close(); err != nil { + t.Fatal(err) + } + reopened, err := Open(context.Background(), Config{Root: s.root, Listen: s.listen, Vault: s.vault, MCPQueue: s.mcpQueue}) + if err != nil { + t.Fatal(err) + } + defer reopened.Close() + status, _ = reopened.StatusForClient(id) + if !status.Ready { + t.Fatalf("restart lost durable trust: %#v", status) + } +} + +func TestSustainableBindingChangesInvalidate(t *testing.T) { + for _, change := range []string{"callback", "secret", "unsubscribe", "auth"} { + t.Run(change, func(t *testing.T) { + s, _, id := openTestBridge(t) + proveSustainable(t, s, id) + p := testEventsParams(id) + switch change { + case "auth": + s.mu.Lock(); policy := s.state.Clients[id]; policy.TokenHash = hashToken("replacement-credential"); s.state.Clients[id] = policy; s.mu.Unlock() + case "callback": + p.Delivery.URL += "&changed=1" + case "secret": + p.Delivery.Secret = "whsec_MTIzNDU2Nzg5MDEyMzQ1Njc4OTAxMjM0NTY3ODkwMTI=" + case "unsubscribe": + if err := s.EventsUnsubscribe(context.Background(), id, p); err != nil { + t.Fatal(err) + } + } + if _, err := s.EventsSubscribe(context.Background(), id, p); err != nil { + t.Fatal(err) + } + status, _ := s.StatusForClient(id) + if status.Ready { + t.Fatal("changed binding retained proof") + } + }) + } +} diff --git a/internal/dotbridge/task_mode_test.go b/internal/dotbridge/task_mode_test.go index 20b3fb6..0a61f51 100644 --- a/internal/dotbridge/task_mode_test.go +++ b/internal/dotbridge/task_mode_test.go @@ -272,7 +272,7 @@ func TestTaskModeHTTPIndependentProofs(t *testing.T) { f.status(t, false, false) f.paused.Store(false) f.subscribe(t) - f.status(t, false, false) + f.status(t, true, true) } func TestTaskModeHTTPUnsuccessfulProbes(t *testing.T) { for _, outcome := range []string{"cancelled", "failed", "wrong_nonce", "subscription_rotated"} { @@ -289,6 +289,7 @@ func TestTaskModeHTTPUnsuccessfulProbes(t *testing.T) { case "wrong_nonce": r.Text = "wrong" case "subscription_rotated": + f.rpc("events/unsubscribe", map[string]any{"name": dotevents.EventName, "arguments": map[string]any{"queue_id": f.pair.QueueID}, "delivery": map[string]any{"mode": "webhook", "url": "https://callback.example/hook"}}) f.subscribe(t) } f.complete(t, c, r) diff --git a/internal/dotbridge/task_proof_test.go b/internal/dotbridge/task_proof_test.go index 1c6d343..e0172ef 100644 --- a/internal/dotbridge/task_proof_test.go +++ b/internal/dotbridge/task_proof_test.go @@ -43,6 +43,8 @@ func testProbeReplay(t *testing.T, mode string) { proof := policy.proof(mode) proof.Verified = past proof.Expires = past.Add(time.Minute) + // Exercise fail-closed legacy proof: no durable binding evidence. + proof.TrustEpoch = "" policy.setProof(mode, proof) s.state.Clients[id] = policy if err = s.saveState(); err != nil { @@ -140,7 +142,7 @@ func TestProbeStatusRenewalCannotMixGenerations(t *testing.T) { } s.events = renewingStatusEvents{EventsAPI: s.events, renew: func() { VerifyTestSubscription(t, s, id) }} status, _ := s.StatusForClient(id) - if status.Ready { - t.Fatal("status combined old proof with renewed subscription") + if !status.Ready { + t.Fatal("continuous same-binding renewal lost verified proof") } } diff --git a/internal/dotbridge/types.go b/internal/dotbridge/types.go index 33a8765..7c7f55a 100644 --- a/internal/dotbridge/types.go +++ b/internal/dotbridge/types.go @@ -22,12 +22,14 @@ type ClientPolicy struct { ProbeGeneration uint64 `json:"probe_generation,omitempty"` ProbeSubscriptionExpires time.Time `json:"probe_subscription_expires_at,omitempty"` ProbeConsumed bool `json:"probe_consumed,omitempty"` + ProbeTrustEpoch string `json:"probe_trust_epoch,omitempty"` TaskProbe ProbeProof `json:"task_probe,omitempty"` } // ProbeProof is bounded private state for one mode's latest challenge. // Consumed survives expiry and subscription rotation; retries cannot renew proof. type ProbeProof struct { + TrustEpoch string `json:"trust_epoch,omitempty"` JobID string `json:"job_id,omitempty"` AnswerHash string `json:"answer_hash,omitempty"` Verified time.Time `json:"verified_at,omitempty"` @@ -41,13 +43,14 @@ func (p ClientPolicy) proof(mode string) ProbeProof { if mode == "task" { return p.TaskProbe } - return ProbeProof{JobID: p.ProbeJobID, AnswerHash: p.ProbeAnswer, Verified: p.ProbeVerified, Expires: p.ProbeExpires, Generation: p.ProbeGeneration, SubscriptionExpires: p.ProbeSubscriptionExpires, Consumed: p.ProbeConsumed} + return ProbeProof{TrustEpoch: p.ProbeTrustEpoch, JobID: p.ProbeJobID, AnswerHash: p.ProbeAnswer, Verified: p.ProbeVerified, Expires: p.ProbeExpires, Generation: p.ProbeGeneration, SubscriptionExpires: p.ProbeSubscriptionExpires, Consumed: p.ProbeConsumed} } func (p *ClientPolicy) setProof(mode string, proof ProbeProof) { if mode == "task" { p.TaskProbe = proof return } + p.ProbeTrustEpoch = proof.TrustEpoch p.ProbeJobID, p.ProbeAnswer = proof.JobID, proof.AnswerHash p.ProbeVerified, p.ProbeExpires = proof.Verified, proof.Expires p.ProbeGeneration, p.ProbeSubscriptionExpires, p.ProbeConsumed = proof.Generation, proof.SubscriptionExpires, proof.Consumed @@ -59,9 +62,10 @@ type CapPolicy struct { } type PersistedState struct { - Version int `json:"version"` - Enabled bool `json:"enabled"` - Clients map[string]ClientPolicy `json:"clients"` + Version int `json:"version"` + Enabled bool `json:"enabled"` + Clients map[string]ClientPolicy `json:"clients"` + Subscriptions map[string]subscriptionProof `json:"trusted_subscriptions,omitempty"` } // CapacityDTO reports global retained job counts for operator visibility. @@ -87,8 +91,11 @@ type StatusDTO struct { } type SubscriptionDTO struct { - State string `json:"state"` - ExpiresAt *time.Time `json:"expires_at,omitempty"` + RefreshBefore *time.Time `json:"refresh_before,omitempty"` + LastRefreshedAt *time.Time `json:"last_refreshed_at,omitempty"` + RenewalState string `json:"renewal_state,omitempty"` + State string `json:"state"` + ExpiresAt *time.Time `json:"expires_at,omitempty"` } // SubscribeFailureDTO reports the most recent refused subscription attempt diff --git a/internal/dotbridge/worker.go b/internal/dotbridge/worker.go index 2f73801..60a6033 100644 --- a/internal/dotbridge/worker.go +++ b/internal/dotbridge/worker.go @@ -149,10 +149,13 @@ func (s *Service) completeTask(jobID, token string, result dotqueue.Result) (dot return dotqueue.Job{}, errors.New("bridge_paused") } proof := policy.proof(job.Mode) - if s.state.Enabled && active && sub.Generation == b.Generation && proof.Generation == b.Generation && !sub.Expires.After(proof.SubscriptionExpires) && proof.JobID == jobID && proof.AnswerHash == hashToken(result.Text) && !proof.Consumed && proof.Verified.IsZero() && job.State == dotqueue.StateCompleted && (job.Mode == dotqueue.ModeAnswer || job.Mode == dotqueue.ModeTask) { + if s.state.Enabled && active && sub.Generation == b.Generation && proof.Generation == b.Generation && (proof.TrustEpoch != "" && proof.TrustEpoch == sub.TrustEpoch || !sub.Expires.After(proof.SubscriptionExpires)) && proof.JobID == jobID && proof.AnswerHash == hashToken(result.Text) && !proof.Consumed && proof.Verified.IsZero() && job.State == dotqueue.StateCompleted && (job.Mode == dotqueue.ModeAnswer || job.Mode == dotqueue.ModeTask) { // Use the durable queue terminal timestamp, never the retry time. proof.Verified = job.UpdatedAt proof.Expires = job.UpdatedAt.Add(probeValidity) + if proof.TrustEpoch != "" { + proof.Expires = sub.Expires + } proof.Consumed = true proof.SubscriptionExpires = sub.Expires if sub.Expires.Before(proof.Expires) { diff --git a/internal/dotevents/manager.go b/internal/dotevents/manager.go index 1b8948a..f0e5469 100644 --- a/internal/dotevents/manager.go +++ b/internal/dotevents/manager.go @@ -65,14 +65,18 @@ func (m *Manager) subscribe(ctx context.Context, principal string, p Params, clo if e != nil { return Subscription{}, e } - now := clock().UTC() var previous *subscriptionRecord var cached bool - rec := &subscriptionRecord{ID: id, Generation: gen, Principal: principal, Params: p, Secret: p.Delivery.Secret, ExpiresAt: now.Add(time.Duration(ttl) * time.Millisecond), CreatedAt: now, UpdatedAt: now} + rec := &subscriptionRecord{TrustEpoch: gen, ID: id, Generation: gen, Principal: principal, Params: p, Secret: p.Delivery.Secret} if e = m.store.withLock(false, true, func(st *persistenceState) error { if !st.Enabled { return errf("disabled") } + // Eligibility and the new grant start at the serialized mutation, not + // request arrival: acquiring the store lock may outlive the old lease. + now := clock().UTC() + rec.ExpiresAt = now.Add(time.Duration(ttl) * time.Millisecond) + rec.CreatedAt, rec.UpdatedAt = now, now for k, s := range st.Subscriptions { if !s.ExpiresAt.After(now) { delete(st.Subscriptions, k) @@ -97,6 +101,11 @@ func (m *Manager) subscribe(ctx context.Context, principal string, p Params, clo return errf("capacity") } if old != nil { + // Only the exact continuously authorized binding retains mode trust. + // Secret replacement deliberately starts a new epoch, even on a cache hit. + if old.CallbackVerified && old.TrustEpoch != "" && old.Secret == rec.Secret { + rec.TrustEpoch = old.TrustEpoch + } rec.CreatedAt = old.CreatedAt if old.Secret != rec.Secret { rec.PreviousSecret = old.Secret @@ -106,13 +115,32 @@ func (m *Manager) subscribe(ctx context.Context, principal string, p Params, clo rec.RotateUntil = old.RotateUntil } } + if cached && st.LastSubscribe != nil && st.LastSubscribe.Principal == principal { + st.LastSubscribe = nil + } st.Subscriptions[id] = rec return nil }); e != nil { return Subscription{}, e } if cached { - return Subscription{ID: id, ExpiresAt: rec.ExpiresAt, CallbackVerified: true}, nil + err := m.store.withLock(false, true, func(st *persistenceState) error { + now := clock().UTC() + s := st.Subscriptions[id] + if !st.Enabled || s == nil || s.Generation != gen || !s.CallbackVerified || !s.ExpiresAt.After(now) { + return errf("stale") + } + // The cached path also reacquires the lock after persistence. A + // delayed finalization cannot carry trust past the source lease. + if previous != nil && !previous.ExpiresAt.After(now) { + s.TrustEpoch, rec.TrustEpoch = gen, gen + } + return nil + }) + if err != nil { + return Subscription{}, err + } + return Subscription{TrustEpoch: rec.TrustEpoch, ID: id, ExpiresAt: rec.ExpiresAt, CallbackVerified: true}, nil } if e = m.delivery.verify(ctx, rec); e != nil { // Restore a previously accepted refresh only if no revoke/refresh won @@ -148,19 +176,27 @@ func (m *Manager) subscribe(ctx context.Context, principal string, p Params, clo if !st.Enabled { return errf("disabled") } + now := clock().UTC() s := st.Subscriptions[id] - if s == nil || s.Generation != gen || !s.ExpiresAt.After(clock()) { + if s == nil || s.Generation != gen || !s.ExpiresAt.After(now) { return errf("stale") } + if previous != nil && !previous.ExpiresAt.After(now) { + s.TrustEpoch = gen + rec.TrustEpoch = gen + } + if st.LastSubscribe != nil && st.LastSubscribe.Principal == principal { + st.LastSubscribe = nil + } s.CallbackVerified = true - s.UpdatedAt = clock().UTC() + s.UpdatedAt = now s.VerifiedUntil = boundedVerificationDeadline(s.UpdatedAt.Add(verifiedCacheTTL), s.ExpiresAt, s.ExpiresAt, s.UpdatedAt) return nil }) if e != nil { return Subscription{}, e } - return Subscription{ID: id, ExpiresAt: rec.ExpiresAt, CallbackVerified: true}, nil + return Subscription{TrustEpoch: rec.TrustEpoch, ID: id, ExpiresAt: rec.ExpiresAt, CallbackVerified: true}, nil } // diagnosticReason accepts only stable categories, never arbitrary error text. @@ -390,6 +426,9 @@ func (m *Manager) Status(principal, q string) (Status, error) { if s.Principal == principal && s.ExpiresAt.After(time.Now()) { out.Subscribed = true out.CallbackVerified = out.CallbackVerified || s.CallbackVerified + if s.CallbackVerified && s.TrustEpoch != "" { + out.Bindings = append(out.Bindings, TrustedBinding{ID: s.ID, Epoch: s.TrustEpoch, Expires: s.ExpiresAt, Refreshed: s.UpdatedAt}) + } } } } @@ -398,7 +437,7 @@ func (m *Manager) Status(principal, q string) (Status, error) { out.Acknowledged = r.Acknowledged out.LastCode = r.LastCode } - if f := st.LastSubscribe; f != nil && f.Principal == principal && !out.CallbackVerified { + if f := st.LastSubscribe; f != nil && f.Principal == principal { out.SubscribeFailure = &SubscribeFailure{ At: f.At.UTC().Format(time.RFC3339), Reason: f.Reason, diff --git a/internal/dotevents/state.go b/internal/dotevents/state.go index eda2a85..3444f96 100644 --- a/internal/dotevents/state.go +++ b/internal/dotevents/state.go @@ -16,6 +16,7 @@ const ( ) type subscriptionRecord struct { + TrustEpoch string `json:"trust_epoch,omitempty"` VerifiedUntil time.Time `json:"verified_until"` Generation string `json:"generation"` ID string `json:"id"` @@ -98,6 +99,7 @@ func parsePersistenceState(data []byte) (*persistenceState, error) { } } for id, sub := range st.Subscriptions { + if sub != nil && sub.TrustEpoch != "" && validateID(sub.TrustEpoch) != nil { return nil, errf("corrupt") } if sub != nil && !sub.VerifiedUntil.IsZero() { // Validate the persisted shape, not elapsed wall time. Legacy // UpdatedAt predates lock/save/HTTP work, and wall clocks can step diff --git a/internal/dotevents/store.go b/internal/dotevents/store.go index 7933416..5004609 100644 --- a/internal/dotevents/store.go +++ b/internal/dotevents/store.go @@ -15,6 +15,8 @@ import ( type store struct { dir string key []byte + // beforeLock permits deterministic lock-contention tests; nil in production. + beforeLock func() } func openStore(dir string, key []byte) (*store, error) { @@ -50,6 +52,9 @@ func (s *store) withLock(allowCreate bool, mutate bool, fn func(*persistenceStat lock := flock.New(lockPath) ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() + if s.beforeLock != nil { + s.beforeLock() + } ok, err := lock.TryLockContext(ctx, 10*time.Millisecond) if err != nil || !ok { return errf("unknown") diff --git a/internal/dotevents/trust.go b/internal/dotevents/trust.go new file mode 100644 index 0000000..bd96334 --- /dev/null +++ b/internal/dotevents/trust.go @@ -0,0 +1,8 @@ +package dotevents + +// TrustedBindings returns only verified grants for this authenticated queue. +// This internal API must never be serialized into the public status response. +func (m *Manager) TrustedBindings(principal string) ([]TrustedBinding, error) { + status, err := m.Status(principal, principal) + return status.Bindings, err +} diff --git a/internal/dotevents/trust_failure_test.go b/internal/dotevents/trust_failure_test.go new file mode 100644 index 0000000..3f8f573 --- /dev/null +++ b/internal/dotevents/trust_failure_test.go @@ -0,0 +1,51 @@ +package dotevents + +import ( + "context" + "net/http" + "sync/atomic" + "testing" + "time" +) + +func TestFailedRefreshKeepsOldGrantAndReportsFailure(t *testing.T) { + var fail atomic.Bool + m, _, stop, _ := withFixtureServerURL(t, func(w http.ResponseWriter, r *http.Request) { + if fail.Load() { + w.WriteHeader(500) + return + } + echoVerification(w, r) + }) + defer stop() + p := testParams("alice") + first, err := m.Subscribe(context.Background(), "alice", p) + if err != nil { + t.Fatal(err) + } + if err = m.store.withLock(false, true, func(st *persistenceState) error { st.Subscriptions[first.ID].VerifiedUntil = time.Time{}; return nil }); err != nil { + t.Fatal(err) + } + fail.Store(true) + if _, err = m.Subscribe(context.Background(), "alice", p); err == nil { + t.Fatal("refresh unexpectedly succeeded") + } + status, err := m.Status("alice", "alice") + if err != nil { + t.Fatal(err) + } + if !status.Subscribed || !status.CallbackVerified || status.SubscribeFailure == nil { + t.Fatalf("missing active-lease failure diagnostics: %#v", status) + } + if len(status.Bindings) != 1 || status.Bindings[0].Epoch != first.TrustEpoch || !status.Bindings[0].Expires.Equal(first.ExpiresAt) { + t.Fatal("failed refresh altered authorized grant") + } + fail.Store(false) + if _, err = m.Subscribe(context.Background(), "alice", p); err != nil { + t.Fatal(err) + } + status, err = m.Status("alice", "alice") + if err != nil || status.SubscribeFailure != nil { + t.Fatal("successful refresh did not clear failure") + } +} diff --git a/internal/dotevents/trust_lock_test.go b/internal/dotevents/trust_lock_test.go new file mode 100644 index 0000000..0529a09 --- /dev/null +++ b/internal/dotevents/trust_lock_test.go @@ -0,0 +1,67 @@ +package dotevents + +import ( + "context" + "net/http" + "sync/atomic" + "testing" + "time" +) + +func TestCachedRefreshLockWaitCannotRetainExpiredEpoch(t *testing.T) { + for _, point := range []string{"mutation", "final"} { + t.Run(point, func(t *testing.T) { + base := time.Now().UTC() + m, dir, stop, _ := withFixtureServerURL(t, func(w http.ResponseWriter, r *http.Request) { echoVerification(w, r) }) + defer stop() + p := testParams("alice") + p.TTL = []byte("60000") + first, err := m.subscribe(context.Background(), "alice", p, func() time.Time { return base }) + if err != nil { + t.Fatal(err) + } + holder, err := openStore(dir, testKey) + if err != nil { + t.Fatal(err) + } + var seconds atomic.Int64 + seconds.Store(59) + var lockCalls atomic.Int32 + reached := make(chan struct{}) + releaseAttempt := make(chan struct{}) + target := int32(1) + if point == "final" { + target = 2 + } + m.store.beforeLock = func() { + if lockCalls.Add(1) == target { + close(reached) + <-releaseAttempt + } + } + done := make(chan struct{}) + var renewed Subscription + var subscribeErr error + go func() { + defer close(done) + renewed, subscribeErr = m.subscribe(context.Background(), "alice", p, func() time.Time { return base.Add(time.Duration(seconds.Load()) * time.Second) }) + }() + <-reached + // Hold the real store lock while the renewal has reached lock acquisition. + err = holder.withLock(false, false, func(*persistenceState) error { seconds.Store(61); close(releaseAttempt); return nil }) + if err != nil { + t.Fatal(err) + } + <-done + if subscribeErr != nil { + t.Fatal(subscribeErr) + } + if renewed.TrustEpoch == first.TrustEpoch { + t.Fatal("cached renewal retained trust across lock-wait lease gap") + } + if point == "mutation" && !renewed.ExpiresAt.Equal(base.Add(121*time.Second)) { + t.Fatalf("grant used pre-lock time: %s", renewed.ExpiresAt) + } + }) + } +} diff --git a/internal/dotevents/trust_test.go b/internal/dotevents/trust_test.go new file mode 100644 index 0000000..33bb01c --- /dev/null +++ b/internal/dotevents/trust_test.go @@ -0,0 +1,71 @@ +package dotevents + +import ( + "context" + "net/http" + "testing" + "time" +) + +func TestTrustEpochContinuousLeaseAndGap(t *testing.T) { + m, dir, stop, _ := withFixtureServerURL(t, func(w http.ResponseWriter, r *http.Request) { echoVerification(w, r) }) + defer stop() + now := time.Now().UTC() + p := testParams("alice") + p.TTL = []byte("3600000") + first, err := m.subscribe(context.Background(), "alice", p, func() time.Time { return now }) + if err != nil { + t.Fatal(err) + } + if first.TrustEpoch == "" { + t.Fatal("verified grant missing trust epoch") + } + reopened, err := Open(dir, testKey, m.delivery) + if err != nil { + t.Fatal(err) + } + renewed, err := reopened.subscribe(context.Background(), "alice", p, func() time.Time { return now.Add(20 * time.Minute) }) + if err != nil { + t.Fatal(err) + } + if renewed.TrustEpoch != first.TrustEpoch || !renewed.ExpiresAt.After(first.ExpiresAt) { + t.Fatal("renewal lost durable epoch or failed to extend requested grant") + } + gap, err := reopened.subscribe(context.Background(), "alice", p, func() time.Time { return renewed.ExpiresAt.Add(time.Second) }) + if err != nil { + t.Fatal(err) + } + if gap.TrustEpoch == renewed.TrustEpoch { + t.Fatal("lease gap retained trust") + } +} + +func TestTrustEpochSlowRefreshCrossesOldExpiry(t *testing.T) { + now := time.Now().UTC() + m, _, stop, _ := withFixtureServerURL(t, func(w http.ResponseWriter, r *http.Request) { echoVerification(w, r) }) + defer stop() + p := testParams("alice") + p.TTL = []byte("60000") + first, err := m.subscribe(context.Background(), "alice", p, func() time.Time { return now }) + if err != nil { + t.Fatal(err) + } + // Expire only the callback verification cache to force actual callback IO. + if err = m.store.withLock(false, true, func(st *persistenceState) error { st.Subscriptions[first.ID].VerifiedUntil = time.Time{}; return nil }); err != nil { + t.Fatal(err) + } + calls := 0 + second, err := m.subscribe(context.Background(), "alice", p, func() time.Time { + calls++ + if calls == 1 { + return now.Add(59 * time.Second) + } + return now.Add(61 * time.Second) + }) + if err != nil { + t.Fatal(err) + } + if first.TrustEpoch == second.TrustEpoch { + t.Fatal("slow verification resurrected expired trust") + } +} diff --git a/internal/dotevents/types.go b/internal/dotevents/types.go index 3f507de..c28de6a 100644 --- a/internal/dotevents/types.go +++ b/internal/dotevents/types.go @@ -26,6 +26,7 @@ type Params struct { } type Subscription struct { + TrustEpoch string `json:"-"` ID string `json:"id"` ExpiresAt time.Time `json:"expiresAt"` CallbackVerified bool `json:"callbackVerified"` @@ -47,13 +48,21 @@ type Notice struct { Mode string } +// TrustedBinding is internal evidence from encrypted subscription state. It +// intentionally carries neither the callback URL nor any credential. +type TrustedBinding struct { + ID, Epoch string + Expires, Refreshed time.Time +} + type Status struct { - Subscribed bool `json:"subscribed"` - CallbackVerified bool `json:"callback_verified"` - DotReady bool `json:"dot_ready"` - Attempts int `json:"attempts"` - Acknowledged bool `json:"acknowledged"` - LastCode string `json:"last_code,omitempty"` + Bindings []TrustedBinding `json:"-"` + Subscribed bool `json:"subscribed"` + CallbackVerified bool `json:"callback_verified"` + DotReady bool `json:"dot_ready"` + Attempts int `json:"attempts"` + Acknowledged bool `json:"acknowledged"` + LastCode string `json:"last_code,omitempty"` // SubscribeFailure explains the most recent refused subscription once it // is no longer active. It carries no secret or callback URL. SubscribeFailure *SubscribeFailure `json:"subscribe_failure,omitempty"` From df4af32f860389b80f1a6c3ec60d5fc7c9c9ff8e Mon Sep 17 00:00:00 2001 From: Pyuyi <204892921+VeraPyuyi@users.noreply.github.com> Date: Wed, 21 Oct 2026 12:53:00 +0800 Subject: [PATCH 42/47] fix(bridge): report verified legacy subscription leases after upgrade --- internal/dotbridge/legacy_status_test.go | 184 +++++++++++++++++++++++ internal/dotbridge/service.go | 11 ++ internal/dotevents/legacy_status_test.go | 105 +++++++++++++ internal/dotevents/manager.go | 13 +- internal/dotevents/types.go | 3 +- 5 files changed, 313 insertions(+), 3 deletions(-) create mode 100644 internal/dotbridge/legacy_status_test.go create mode 100644 internal/dotevents/legacy_status_test.go diff --git a/internal/dotbridge/legacy_status_test.go b/internal/dotbridge/legacy_status_test.go new file mode 100644 index 0000000..8840538 --- /dev/null +++ b/internal/dotbridge/legacy_status_test.go @@ -0,0 +1,184 @@ +package dotbridge + +import ( + "bytes" + "context" + "crypto/aes" + "crypto/cipher" + "crypto/rand" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/dotcontrol/dotcontrol/internal/dotevents" + "github.com/dotcontrol/dotcontrol/internal/dotqueue" +) + +func TestLegacyLeaseStatusAndAuthenticatedRenewalAfterRestart(t *testing.T) { + s, token, id, legacy := openLegacyStatusBridge(t) + eventsPath := filepath.Join(s.root, "events", "events.state.de") + bridgePath := filepath.Join(s.root, "bridge.state.json") + eventsBefore, bridgeBefore := mustRead(eventsPath), mustRead(bridgePath) + policy := s.state.Clients[id] + api := httptest.NewServer(s.Handler()) + defer api.Close() + + code, raw := listHTTPRequest(t, api, http.MethodGet, "/dot-bridge/v1/status", token, nil) + var status StatusDTO + if code != http.StatusOK || json.Unmarshal(raw, &status) != nil { + t.Fatalf("status: %d %s", code, raw) + } + if status.Subscription.State != "active" || status.Subscription.ExpiresAt == nil || !status.Subscription.ExpiresAt.Equal(legacy.Expires) || status.Subscription.RefreshBefore == nil || !status.Subscription.RefreshBefore.Equal(legacy.Expires) || status.Subscription.LastRefreshedAt == nil || !status.Subscription.LastRefreshedAt.Equal(legacy.Refreshed) { + t.Errorf("verified legacy lease missing from public status: %#v", status.Subscription) + } + if status.SubscribeFailure != nil || status.Subscription.RenewalState != "current" { + t.Errorf("obsolete failure still reported: %#v", status) + } + if status.Ready || status.Code != "probe_required" || status.Capabilities.Text || status.Capabilities.Tasks || status.VerifiedAt != nil || status.ExpiresAt != nil { + t.Errorf("legacy proof became ready after restart: %#v", status) + } + for _, private := range []string{testEventsSecret(), fixtureCallbackURL, legacy.ID, policy.TokenHash} { + if strings.Contains(string(raw), private) { + t.Error("public status leaked private binding data") + } + } + assertLegacyWorkRefused(t, s, id) + if _, err := s.StartProbe(id); err == nil || err.Error() != "subscription_required" { + t.Fatalf("probe adopted unauthenticated legacy binding: %v", err) + } + if len(s.subscriptions) != 0 || s.state.Clients[id] != policy || !bytes.Equal(eventsBefore, mustRead(eventsPath)) || !bytes.Equal(bridgeBefore, mustRead(bridgePath)) { + t.Fatal("read or rejected work mutated legacy authorization") + } + + // Only a real authenticated eventsSubscribe may create the new epoch and + // bridge binding. Even an unexpired consumed legacy proof cannot follow it. + renewed, err := s.EventsSubscribe(context.Background(), s.mcpQueue, testEventsParams(id)) + if err != nil { + t.Fatal(err) + } + if !renewed.CallbackVerified || renewed.TrustEpoch == "" || renewed.ID != legacy.ID { + t.Fatalf("authenticated renewal did not upgrade legacy grant: %#v", renewed) + } + sub := s.subscriptions[id] + if sub.TrustEpoch != renewed.TrustEpoch || sub.BindingID != legacy.ID || sub.AuthHash != policy.TokenHash || sub.Consumer != s.mcpQueue { + t.Fatalf("renewal missing authenticated binding: %#v", sub) + } + status, err = s.StatusForClient(id) + if err != nil || status.Ready || status.Code != "probe_required" || status.Subscription.State != "active" || status.Capabilities.Text || status.Capabilities.Tasks { + t.Fatalf("renewal promoted legacy proof: %#v %v", status, err) + } + assertLegacyWorkRefused(t, s, id) + proveSustainable(t, s, id) + status, err = s.StatusForClient(id) + if err != nil || !status.Ready || !status.Capabilities.Text || status.Capabilities.Tasks { + t.Fatalf("fresh answer probe did not establish only answer capability: %#v %v", status, err) + } +} + +func assertLegacyWorkRefused(t *testing.T, s *Service, id string) { + t.Helper() + before, err := s.queue.List(id) + if err != nil { + t.Fatal(err) + } + for _, mode := range []string{dotqueue.ModeAnswer, dotqueue.ModeTask} { + req := SubmitRequest{RequestID: "legacy." + mode, Mode: mode, SessionID: "legacy", Task: "must not run"} + if mode == dotqueue.ModeAnswer { + req.Task = "" + req.Messages = []MessageDTO{{Role: "user", Content: "must not run"}} + } + _, _, err := s.submit(s.state.Clients[id], req) + want := "probe_required" + if mode == dotqueue.ModeTask { + want = "task_probe_required" + } + if err == nil || err.Error() != want { + t.Errorf("legacy %s submission: %v, want %s", mode, err, want) + } + } + after, err := s.queue.List(id) + if err != nil || len(after) != len(before) { + t.Fatalf("unready submission created jobs: before=%d after=%d err=%v", len(before), len(after), err) + } +} + +// Build only temporary encrypted state. Relative times match a grant created at +// 01:45, renewed at 04:02, and inspected after a 04:22 upgrade with a 05:02 expiry. +func openLegacyStatusBridge(t *testing.T) (*Service, string, string, dotevents.TrustedBinding) { + t.Helper() + s, token, id := openTestBridge(t) + proveSustainable(t, s, id) + if err := s.Close(); err != nil { + t.Fatal(err) + } + now := time.Now().UTC().Truncate(time.Second) + legacy := dotevents.TrustedBinding{ID: s.subscriptions[id].BindingID, Refreshed: now.Add(-20 * time.Minute), Expires: now.Add(40*time.Minute - time.Second)} + policy := s.state.Clients[id] + proof := policy.proof(dotqueue.ModeAnswer) + proof.TrustEpoch = "" + proof.Verified, proof.Expires = now.Add(-time.Minute), now.Add(9*time.Minute) + proof.SubscriptionExpires = legacy.Expires + policy.setProof(dotqueue.ModeAnswer, proof) + policy.setProof(dotqueue.ModeTask, proof) + policy.Capabilities = CapPolicy{Text: true, Tasks: true} + s.state.Clients[id] = policy + delete(s.subscriptions, id) // Old binaries never persisted this ephemeral map. + if err := s.saveState(); err != nil { + t.Fatal(err) + } + + p := testEventsParams(id) + p.TTL = []byte("3600000") + plain, err := json.Marshal(map[string]any{ + "state_version": 1, "enabled": true, + "subscriptions": map[string]any{legacy.ID: map[string]any{ + "id": legacy.ID, "principal": id, "params": p, "secret": p.Delivery.Secret, + "generation": "legacy-generation", "callback_verified": true, + "created_at": now.Add(-157 * time.Minute), "updated_at": legacy.Refreshed, + "expires_at": legacy.Expires, "verified_until": legacy.Refreshed.Add(5 * time.Minute), + }}, + "deliveries": map[string]any{}, "queue_status": map[string]any{}, + "last_subscribe": map[string]any{"at": now.Add(-65 * time.Minute), "principal": id, "reason": "network", "stage": "network"}, + }) + if err != nil { + t.Fatal(err) + } + encoded, err := s.vault.Get(StateKeyName) + if err != nil { + t.Fatal(err) + } + key, err := decodeStateKey(encoded) + if err != nil { + t.Fatal(err) + } + block, err := aes.NewCipher(key) + if err != nil { + t.Fatal(err) + } + gcm, err := cipher.NewGCM(block) + if err != nil { + t.Fatal(err) + } + nonce := make([]byte, gcm.NonceSize()) + if _, err := rand.Read(nonce); err != nil { + t.Fatal(err) + } + sealed := append([]byte{'E', 1}, nonce...) + sealed = gcm.Seal(sealed, nonce, plain, nil) + if err := os.WriteFile(filepath.Join(s.root, "events", "events.state.de"), sealed, 0600); err != nil { + t.Fatal(err) + } + delivery, stop := NewTestEventsDelivery(t) + t.Cleanup(stop) + reopened, err := Open(context.Background(), Config{Root: s.root, Listen: s.listen, Vault: s.vault, MCPQueue: s.mcpQueue, EventsDelivery: delivery}) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = reopened.Close() }) + return reopened, token, id, legacy +} diff --git a/internal/dotbridge/service.go b/internal/dotbridge/service.go index 5c5f0ae..750c845 100644 --- a/internal/dotbridge/service.go +++ b/internal/dotbridge/service.go @@ -274,6 +274,17 @@ func (s *Service) statusDTO(policy ClientPolicy) StatusDTO { out := StatusDTO{Version: APIVersion, Enabled: s.state.Enabled, Paired: exists && !current.Revoked, Code: "probe_required", Subscription: SubscriptionDTO{State: "inactive"}} if active { out.Subscription = SubscriptionDTO{State: "active", ExpiresAt: &sub.Expires, RefreshBefore: &sub.Expires, LastRefreshedAt: &sub.Refreshed, RenewalState: "current"} + } else if sub.Generation == 0 && sub.Expires.IsZero() && e == nil && gateOK && exists && !current.Revoked && current.TokenHash == policy.TokenHash && s.capacityAllowedLocked() { + // Old binaries did not persist the bridge subscription map. Report the + // actual verified legacy lease without adopting it as authorization. + // Only an authenticated EventsSubscribe can create the new trust record; + // active stays false so consumed legacy proofs cannot authorize work. + for _, b := range eventStatus.Bindings { + if b.Epoch == "" && now.Before(b.Expires) && (out.Subscription.ExpiresAt == nil || b.Expires.After(*out.Subscription.ExpiresAt) || b.Expires.Equal(*out.Subscription.ExpiresAt) && b.Refreshed.After(*out.Subscription.LastRefreshedAt)) { + expires, refreshed := b.Expires, b.Refreshed + out.Subscription = SubscriptionDTO{State: "active", ExpiresAt: &expires, RefreshBefore: &expires, LastRefreshedAt: &refreshed, RenewalState: "current"} + } + } } if eventStatus.SubscribeFailure != nil && gateOK && exists && !current.Revoked && current.TokenHash == policy.TokenHash && s.capacityAllowedLocked() { // A refused subscription is not silently "inactive": expose why the diff --git a/internal/dotevents/legacy_status_test.go b/internal/dotevents/legacy_status_test.go new file mode 100644 index 0000000..1dd1926 --- /dev/null +++ b/internal/dotevents/legacy_status_test.go @@ -0,0 +1,105 @@ +package dotevents + +import ( + "bytes" + "os" + "path/filepath" + "testing" + "time" +) + +func TestLegacyVerifiedBindingStatusAfterRestart(t *testing.T) { + m, dir := openTestManager(t) + now := time.Now().UTC().Truncate(time.Second) + refreshed, expires := now.Add(-20*time.Minute), now.Add(40*time.Minute-time.Second) + p := testParams("alice") + p.TTL = []byte("3600000") + id := subscriptionID("alice", p) + if err := m.store.withLock(false, true, func(st *persistenceState) error { + // Pre-epoch binaries persisted callback verification, but could leave + // an earlier failed refresh behind after a subsequent success. + st.Subscriptions[id] = &subscriptionRecord{ + ID: id, Principal: "alice", Params: p, Secret: p.Delivery.Secret, + Generation: "legacy-generation", CallbackVerified: true, + CreatedAt: now.Add(-157 * time.Minute), UpdatedAt: refreshed, + ExpiresAt: expires, VerifiedUntil: refreshed.Add(verifiedCacheTTL), + } + st.LastSubscribe = &subscribeFailure{At: now.Add(-65 * time.Minute), Principal: "alice", Reason: "network", Stage: "network"} + return nil + }); err != nil { + t.Fatal(err) + } + reopened, err := Open(dir, testKey, nil) + if err != nil { + t.Fatal(err) + } + path := filepath.Join(dir, stateFileName) + before, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + status, err := reopened.Status("alice", "alice") + if err != nil { + t.Fatal(err) + } + if !status.Subscribed || !status.CallbackVerified || len(status.Bindings) != 1 { + t.Errorf("verified legacy lease omitted: %#v", status) + } else if b := status.Bindings[0]; b.ID != id || b.Epoch != "" || !b.Expires.Equal(expires) || !b.Refreshed.Equal(refreshed) { + t.Errorf("legacy binding changed: %#v", b) + } + if status.SubscribeFailure != nil { + t.Errorf("failure predating verified renewal is still current: %#v", status.SubscribeFailure) + } + after, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(before, after) { + t.Fatal("status mutated encrypted state") + } +} + +func TestSubscribeFailureOnlySuppressedByLaterVerifiedRenewal(t *testing.T) { + for _, tc := range []struct { + name string + principal string + verified bool + failureAge time.Duration + wantFailure bool + }{ + {"stale", "alice", true, time.Minute, false}, + {"same_time", "alice", true, 0, true}, + {"newer_failure", "alice", true, -time.Minute, true}, + {"unverified", "alice", false, time.Minute, true}, + {"another_queue", "bob", true, time.Minute, true}, + } { + t.Run(tc.name, func(t *testing.T) { + m, _ := openTestManager(t) + now := time.Now().UTC().Truncate(time.Second) + refreshed := now.Add(-20 * time.Minute) + p := testParams(tc.principal) + id := subscriptionID(tc.principal, p) + if err := m.store.withLock(false, true, func(st *persistenceState) error { + st.Subscriptions[id] = &subscriptionRecord{ + ID: id, Principal: tc.principal, Params: p, Secret: p.Delivery.Secret, + Generation: "legacy-generation", CallbackVerified: tc.verified, + CreatedAt: refreshed, UpdatedAt: refreshed, ExpiresAt: now.Add(40 * time.Minute), + } + st.LastSubscribe = &subscribeFailure{At: refreshed.Add(-tc.failureAge), Principal: "alice", Reason: "network", Stage: "network"} + return nil + }); err != nil { + t.Fatal(err) + } + status, err := m.Status("alice", "alice") + if err != nil { + t.Fatal(err) + } + if (status.SubscribeFailure != nil) != tc.wantFailure { + t.Fatalf("current failure: %#v, want present=%v", status.SubscribeFailure, tc.wantFailure) + } + if (!tc.verified || tc.principal != "alice") && len(status.Bindings) != 0 { + t.Fatalf("unverified or foreign binding exposed: %#v", status.Bindings) + } + }) + } +} diff --git a/internal/dotevents/manager.go b/internal/dotevents/manager.go index f0e5469..2d32a01 100644 --- a/internal/dotevents/manager.go +++ b/internal/dotevents/manager.go @@ -421,13 +421,19 @@ func (m *Manager) Status(principal, q string) (Status, error) { } var out Status e := m.store.withLock(false, false, func(st *persistenceState) error { + var verifiedUpdated time.Time if st.Enabled { for _, s := range st.Subscriptions { if s.Principal == principal && s.ExpiresAt.After(time.Now()) { out.Subscribed = true out.CallbackVerified = out.CallbackVerified || s.CallbackVerified - if s.CallbackVerified && s.TrustEpoch != "" { + if s.CallbackVerified { + // Legacy verified leases remain real grants, but an empty + // epoch supplies no evidence of durable mode trust. out.Bindings = append(out.Bindings, TrustedBinding{ID: s.ID, Epoch: s.TrustEpoch, Expires: s.ExpiresAt, Refreshed: s.UpdatedAt}) + if s.UpdatedAt.After(verifiedUpdated) { + verifiedUpdated = s.UpdatedAt + } } } } @@ -437,7 +443,10 @@ func (m *Manager) Status(principal, q string) (Status, error) { out.Acknowledged = r.Acknowledged out.LastCode = r.LastCode } - if f := st.LastSubscribe; f != nil && f.Principal == principal { + // Pre-epoch binaries could retain a failure after a later successful + // refresh. Suppress only diagnostics strictly older than a verified + // active binding for this queue; preserve newer refused renewals. + if f := st.LastSubscribe; f != nil && f.Principal == principal && !verifiedUpdated.After(f.At) { out.SubscribeFailure = &SubscribeFailure{ At: f.At.UTC().Format(time.RFC3339), Reason: f.Reason, diff --git a/internal/dotevents/types.go b/internal/dotevents/types.go index c28de6a..92b5c7d 100644 --- a/internal/dotevents/types.go +++ b/internal/dotevents/types.go @@ -48,7 +48,8 @@ type Notice struct { Mode string } -// TrustedBinding is internal evidence from encrypted subscription state. It +// TrustedBinding is internal evidence from encrypted subscription state. An +// empty Epoch identifies a verified legacy lease, not durable mode trust. It // intentionally carries neither the callback URL nor any credential. type TrustedBinding struct { ID, Epoch string From 80724119d72cbfa36c1125aefd8bdea636255007 Mon Sep 17 00:00:00 2001 From: Pyuyi <204892921+VeraPyuyi@users.noreply.github.com> Date: Wed, 21 Oct 2026 20:50:32 +0800 Subject: [PATCH 43/47] test: compare ACL principals as SIDs and use physical temp roots GitHub CI failed on Windows and macOS while the same packages passed locally, because two fixtures assumed a particular platform environment. Windows: the runner's user is the built-in administrator (RID 500), whose SID Windows renders as the "LA" alias in SDDL text, so an ACL that grants only the current user failed a text comparison. Compare the ACE principal as a SID instead, matching the existing dotqueue check, and keep the protected-DACL, single-entry and full-control assertions. macOS: temporary roots sit under the system /var symlink, which the events and queue stores deliberately reject as an ancestor. Resolve the fixture root with EvalSymlinks, as codexbridge and the installer tests already do. Production symlink rejection is unchanged. Full repository race run passed in 76.61s. --- internal/dotbridge/bridge_test.go | 7 +++- internal/dotevents/paths_windows_test.go | 45 +++++++++++++++--------- internal/dotqueue/store_test.go | 7 +++- 3 files changed, 41 insertions(+), 18 deletions(-) diff --git a/internal/dotbridge/bridge_test.go b/internal/dotbridge/bridge_test.go index 5f007f4..7494440 100644 --- a/internal/dotbridge/bridge_test.go +++ b/internal/dotbridge/bridge_test.go @@ -32,7 +32,12 @@ func openTestBridgeWithPause(t *testing.T, paused func() bool) (*Service, string func openTestBridgeWithGates(t *testing.T, paused, revoked func() bool) (*Service, string, string) { t.Helper() - root := t.TempDir() + // macOS temporary roots sit under the system /var symlink; the bridge + // deliberately rejects symlinked ancestors, so fixtures use physical paths. + root, err := filepath.EvalSymlinks(t.TempDir()) + if err != nil { + t.Fatal(err) + } vault := NewFakeVault() if err := EnsureStateKey(vault); err != nil { t.Fatal(err) diff --git a/internal/dotevents/paths_windows_test.go b/internal/dotevents/paths_windows_test.go index 1a00b47..44e2c09 100644 --- a/internal/dotevents/paths_windows_test.go +++ b/internal/dotevents/paths_windows_test.go @@ -7,10 +7,35 @@ import ( "os" "os/exec" "path/filepath" - "strings" + "runtime" "testing" + "unsafe" ) +// protectedUserACL reports whether sd is a protected DACL with exactly one +// full-control entry for sid. It compares principals as SIDs: Windows renders +// an account SID as a two-letter alias when it matches a well-known RID, so +// text such as "LA" for the built-in administrator (the GitHub Windows runner +// user) describes the same SID this process token holds. +func protectedUserACL(sd *windows.SECURITY_DESCRIPTOR, sid *windows.SID) bool { + flags, _, e := sd.Control() + if e != nil || flags&windows.SE_DACL_PROTECTED == 0 { + return false + } + acl, _, e := sd.DACL() + if e != nil || acl == nil || acl.AceCount != 1 { + return false + } + var ace *windows.ACCESS_ALLOWED_ACE + if e = windows.GetAce(acl, 0, &ace); e != nil || ace == nil { + return false + } + principal := (*windows.SID)(unsafe.Pointer(&ace.SidStart)) + ok := principal.IsValid() && principal.Equals(sid) && ace.Header.AceType == windows.ACCESS_ALLOWED_ACE_TYPE && ace.Mask == windows.STANDARD_RIGHTS_REQUIRED|windows.SYNCHRONIZE|0x1ff + runtime.KeepAlive(sd) + return ok +} + func TestWindowsStoreBoundary(t *testing.T) { for _, p := range []string{`\\server\share\events`, `\\?\C:\events`, `\\.\C:\events`, `C:\events:stream`, `C:\events\NUL`, `C:\events\trailing.`, `C:\events\trailing `} { if _, e := Open(p, testKey, nil); e == nil { @@ -55,11 +80,7 @@ func TestMinorWindowsCreatedAncestors(t *testing.T) { if e != nil { t.Fatal(e) } - acl, _, e := sd.DACL() - if e != nil { - t.Fatal(e) - } - if acl.AceCount != 1 || !strings.Contains(sd.String(), "D:P") || !strings.Contains(sd.String(), user.User.Sid.String()) { + if !protectedUserACL(sd, user.User.Sid) { t.Errorf("new ancestor lacks protected current-user ACL: %s %s", p, sd.String()) } } @@ -89,20 +110,12 @@ func TestWindowsPrivacyEveryTransaction(t *testing.T) { if e != nil { t.Fatal(e) } - acl, _, e := sd.DACL() - if e != nil { - t.Fatal(e) - } user, e := windows.GetCurrentProcessToken().GetTokenUser() if e != nil { t.Fatal(e) } - text := sd.String() - if !strings.Contains(text, user.User.Sid.String()) || strings.Contains(text, ";;;WD") { - t.Errorf("not restricted to current user: %s", text) - } - if acl.AceCount != 1 { - t.Errorf("unexpected ACL entries for %s: %d", p, acl.AceCount) + if !protectedUserACL(sd, user.User.Sid) { + t.Errorf("not restricted to current user: %s", sd.String()) } } } diff --git a/internal/dotqueue/store_test.go b/internal/dotqueue/store_test.go index a3e5871..005f6c9 100644 --- a/internal/dotqueue/store_test.go +++ b/internal/dotqueue/store_test.go @@ -17,7 +17,12 @@ var testKey = []byte{ } func openTestStore(t *testing.T) (*Store, string) { - dir := t.TempDir() + // macOS temporary roots sit under the system /var symlink; the store + // deliberately rejects symlinked ancestors, so fixtures use physical paths. + dir, err := filepath.EvalSymlinks(t.TempDir()) + if err != nil { + t.Fatal(err) + } abs, err := filepath.Abs(dir) if err != nil { t.Fatal(err) From c7734a529d489d655f450812d9dce001a456c651 Mon Sep 17 00:00:00 2001 From: Pyuyi <204892921+VeraPyuyi@users.noreply.github.com> Date: Wed, 21 Oct 2026 21:08:02 +0800 Subject: [PATCH 44/47] test: use physical temp roots for macOS bridge and queue fixtures --- cmd/dotcontrol/dot_bridge_operator_test.go | 2 +- cmd/dotcontrol/dot_bridge_test.go | 15 ++++++++++++++- internal/dotbridge/bridge_test.go | 7 +------ internal/dotbridge/http_flow_test.go | 14 +++++++++++++- internal/dotbridge/pairing_operator_test.go | 2 +- internal/dotbridge/task_mode_test.go | 2 +- internal/dotbridge/testhelpers_test.go | 18 ++++++++++++++++++ internal/dotqueue/store_test.go | 4 ++-- 8 files changed, 51 insertions(+), 13 deletions(-) create mode 100644 internal/dotbridge/testhelpers_test.go diff --git a/cmd/dotcontrol/dot_bridge_operator_test.go b/cmd/dotcontrol/dot_bridge_operator_test.go index 8b1309c..3c3b826 100644 --- a/cmd/dotcontrol/dot_bridge_operator_test.go +++ b/cmd/dotcontrol/dot_bridge_operator_test.go @@ -128,7 +128,7 @@ func TestBridgeOperatorJobsRequestPath(t *testing.T) { _, _ = w.Write([]byte(`{"version":1,"jobs":[]}`)) })) defer srv.Close() - dir := t.TempDir() + dir := physicalTemp(t) bridgeRoot := filepath.Join(dir, "dot-bridge") pairingPath := filepath.Join(bridgeRoot, "pairing-client.fixture.json") if err := os.MkdirAll(bridgeRoot, 0700); err != nil { diff --git a/cmd/dotcontrol/dot_bridge_test.go b/cmd/dotcontrol/dot_bridge_test.go index 077180c..caecd0b 100644 --- a/cmd/dotcontrol/dot_bridge_test.go +++ b/cmd/dotcontrol/dot_bridge_test.go @@ -3,6 +3,7 @@ package main import ( "context" "errors" + "path/filepath" "testing" "github.com/dotcontrol/dotcontrol/internal/config" @@ -16,8 +17,20 @@ func (v *unavailableBridgeVault) Get(string) (string, error) { func (v *unavailableBridgeVault) Set(string, string) error { v.writes++; return nil } func TestOptionalBridgeUnavailableDoesNotReplaceKey(t *testing.T) { v := &unavailableBridgeVault{} - s, e := openRuntimeDotBridge(context.Background(), t.TempDir(), config.Config{Listen: "127.0.0.1:9317"}, v, nil, nil) + s, e := openRuntimeDotBridge(context.Background(), physicalTemp(t), config.Config{Listen: "127.0.0.1:9317"}, v, nil, nil) if e == nil || s != nil || v.writes != 0 { t.Fatalf("service=%v err=%v writes=%d", s, e, v.writes) } } + +// physicalTemp resolves the test's temporary root: macOS temporary directories +// sit under the system /var symlink, which the bridge deliberately rejects as +// an ancestor. Production symlink rejection is unchanged. +func physicalTemp(t *testing.T) string { + t.Helper() + dir, err := filepath.EvalSymlinks(t.TempDir()) + if err != nil { + t.Fatal(err) + } + return dir +} diff --git a/internal/dotbridge/bridge_test.go b/internal/dotbridge/bridge_test.go index 7494440..b24a15b 100644 --- a/internal/dotbridge/bridge_test.go +++ b/internal/dotbridge/bridge_test.go @@ -32,12 +32,7 @@ func openTestBridgeWithPause(t *testing.T, paused func() bool) (*Service, string func openTestBridgeWithGates(t *testing.T, paused, revoked func() bool) (*Service, string, string) { t.Helper() - // macOS temporary roots sit under the system /var symlink; the bridge - // deliberately rejects symlinked ancestors, so fixtures use physical paths. - root, err := filepath.EvalSymlinks(t.TempDir()) - if err != nil { - t.Fatal(err) - } + root := physicalTemp(t) vault := NewFakeVault() if err := EnsureStateKey(vault); err != nil { t.Fatal(err) diff --git a/internal/dotbridge/http_flow_test.go b/internal/dotbridge/http_flow_test.go index ffaf36a..f7c2563 100644 --- a/internal/dotbridge/http_flow_test.go +++ b/internal/dotbridge/http_flow_test.go @@ -51,7 +51,7 @@ func TestReviewHTTPPairedQueueRoundTrip(t *testing.T) { t.Fatal(e) } var paused atomic.Bool - s, e := dotbridge.Open(context.Background(), dotbridge.Config{Root: filepath.Join(t.TempDir(), "bridge"), Listen: "127.0.0.1:9317", Vault: v, MCPQueue: "device.distinct", EventsDelivery: delivery, Paused: paused.Load}) + s, e := dotbridge.Open(context.Background(), dotbridge.Config{Root: filepath.Join(physicalTemp(t), "bridge"), Listen: "127.0.0.1:9317", Vault: v, MCPQueue: "device.distinct", EventsDelivery: delivery, Paused: paused.Load}) if e != nil { t.Fatal(e) } @@ -169,3 +169,15 @@ func TestReviewHTTPPairedQueueRoundTrip(t *testing.T) { t.Fatal("ready while paused") } } + +// physicalTemp resolves the test's temporary root: macOS temporary directories +// sit under the system /var symlink, which the bridge deliberately rejects as +// an ancestor. Production symlink rejection is unchanged. +func physicalTemp(t *testing.T) string { + t.Helper() + dir, err := filepath.EvalSymlinks(t.TempDir()) + if err != nil { + t.Fatal(err) + } + return dir +} diff --git a/internal/dotbridge/pairing_operator_test.go b/internal/dotbridge/pairing_operator_test.go index 64cb13a..0be7984 100644 --- a/internal/dotbridge/pairing_operator_test.go +++ b/internal/dotbridge/pairing_operator_test.go @@ -33,7 +33,7 @@ func TestPairingOperatorRejectHardlink(t *testing.T) { } func TestPairingOperatorAtomicRoundTrip(t *testing.T) { - root := filepath.Join(t.TempDir(), "bridge") + root := filepath.Join(physicalTemp(t), "bridge") original := PairingFile{Version: APIVersion, ClientID: "client.fixture", QueueID: "client.fixture", Token: "fixture-secret", Listen: "127.0.0.1:9317"} if _, err := writePairingFile(root, original); err != nil { t.Fatal(err) diff --git a/internal/dotbridge/task_mode_test.go b/internal/dotbridge/task_mode_test.go index 0a61f51..fcab7b0 100644 --- a/internal/dotbridge/task_mode_test.go +++ b/internal/dotbridge/task_mode_test.go @@ -58,7 +58,7 @@ func newTaskFixture(t *testing.T) *taskFixture { t.Fatal(err) } var err error - f.s, err = dotbridge.Open(context.Background(), dotbridge.Config{Root: filepath.Join(t.TempDir(), "bridge"), Listen: "127.0.0.1:9317", Vault: v, MCPQueue: "device.task-fixture", EventsDelivery: delivery, Paused: f.paused.Load}) + f.s, err = dotbridge.Open(context.Background(), dotbridge.Config{Root: filepath.Join(physicalTemp(t), "bridge"), Listen: "127.0.0.1:9317", Vault: v, MCPQueue: "device.task-fixture", EventsDelivery: delivery, Paused: f.paused.Load}) if err != nil { t.Fatal(err) } diff --git a/internal/dotbridge/testhelpers_test.go b/internal/dotbridge/testhelpers_test.go new file mode 100644 index 0000000..b4be618 --- /dev/null +++ b/internal/dotbridge/testhelpers_test.go @@ -0,0 +1,18 @@ +package dotbridge + +import ( + "path/filepath" + "testing" +) + +// physicalTemp resolves the test's temporary root: macOS temporary directories +// sit under the system /var symlink, which the bridge deliberately rejects as +// an ancestor. Production symlink rejection is unchanged. +func physicalTemp(t *testing.T) string { + t.Helper() + dir, err := filepath.EvalSymlinks(t.TempDir()) + if err != nil { + t.Fatal(err) + } + return dir +} diff --git a/internal/dotqueue/store_test.go b/internal/dotqueue/store_test.go index 005f6c9..9f9181e 100644 --- a/internal/dotqueue/store_test.go +++ b/internal/dotqueue/store_test.go @@ -19,11 +19,11 @@ var testKey = []byte{ func openTestStore(t *testing.T) (*Store, string) { // macOS temporary roots sit under the system /var symlink; the store // deliberately rejects symlinked ancestors, so fixtures use physical paths. - dir, err := filepath.EvalSymlinks(t.TempDir()) + physical, err := filepath.EvalSymlinks(t.TempDir()) if err != nil { t.Fatal(err) } - abs, err := filepath.Abs(dir) + abs, err := filepath.Abs(physical) if err != nil { t.Fatal(err) } From 76b9ec8e7a298d26dd76e1d1531958a088e14419 Mon Sep 17 00:00:00 2001 From: Pyuyi <204892921+VeraPyuyi@users.noreply.github.com> Date: Wed, 21 Oct 2026 21:18:43 +0800 Subject: [PATCH 45/47] test: use physical temp roots for event store fixtures --- internal/dotevents/testutil_test.go | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/internal/dotevents/testutil_test.go b/internal/dotevents/testutil_test.go index 9cb9420..16aeb9e 100644 --- a/internal/dotevents/testutil_test.go +++ b/internal/dotevents/testutil_test.go @@ -7,6 +7,7 @@ import ( "net" "net/http" "net/http/httptest" + "path/filepath" "testing" "time" ) @@ -15,9 +16,21 @@ const fixtureCallbackURL = "https://callback.example/hook?opaque=ok" var testKey = []byte("01234567890123456789012345678901") +// physicalTemp resolves the test's temporary root: macOS temporary directories +// sit under the system /var symlink, which the store deliberately rejects as +// an ancestor. Production symlink rejection is unchanged. +func physicalTemp(t *testing.T) string { + t.Helper() + dir, err := filepath.EvalSymlinks(t.TempDir()) + if err != nil { + t.Fatal(err) + } + return dir +} + func openTestManager(t *testing.T) (*Manager, string) { t.Helper() - dir := t.TempDir() + dir := physicalTemp(t) m, e := Open(dir, testKey, nil) if e != nil { t.Fatal(e) @@ -50,7 +63,7 @@ func withFixtureServerURL(t *testing.T, h http.HandlerFunc) (*Manager, string, f } return (&net.Dialer{Timeout: time.Second}).DialContext(ctx, network, s.Listener.Addr().String()) } - dir := t.TempDir() + dir := physicalTemp(t) m, e := Open(dir, testKey, &Delivery{client: &http.Client{Timeout: maxHTTPTimeout, Transport: tr, CheckRedirect: noRedirect}}) if e != nil { t.Fatal(e) From 94ef5821cf83b10f99ad74a3b30775231a28b474 Mon Sep 17 00:00:00 2001 From: Pyuyi <204892921+VeraPyuyi@users.noreply.github.com> Date: Wed, 21 Oct 2026 21:28:36 +0800 Subject: [PATCH 46/47] test: use physical temp roots in mcp and event fixtures, widen takeover wait --- internal/dotevents/revision_test.go | 2 +- internal/engine/engine_test.go | 2 +- internal/mcpserver/bridge_tools_test.go | 14 +++++++++++++- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/internal/dotevents/revision_test.go b/internal/dotevents/revision_test.go index 41e71b6..89bb4ff 100644 --- a/internal/dotevents/revision_test.go +++ b/internal/dotevents/revision_test.go @@ -62,7 +62,7 @@ func TestReviewIdentityTimestampAndCache(t *testing.T) { } } func TestReviewNewStoreDisabledAndMarker(t *testing.T) { - m, e := Open(t.TempDir(), testKey, nil) + m, e := Open(physicalTemp(t), testKey, nil) if e != nil { t.Fatal(e) } diff --git a/internal/engine/engine_test.go b/internal/engine/engine_test.go index a29e708..2febbe3 100644 --- a/internal/engine/engine_test.go +++ b/internal/engine/engine_test.go @@ -333,7 +333,7 @@ func TestHumanTakeoverCancelsAndReleases(t *testing.T) { if r.State != "unknown" { t.Fatalf("takeover=%s", r.State) } - case <-time.After(time.Second): + case <-time.After(10 * time.Second): t.Fatal("human takeover not cancelled") } if d.releases.Load() == 0 { diff --git a/internal/mcpserver/bridge_tools_test.go b/internal/mcpserver/bridge_tools_test.go index a91de71..09be780 100644 --- a/internal/mcpserver/bridge_tools_test.go +++ b/internal/mcpserver/bridge_tools_test.go @@ -21,7 +21,7 @@ func (bridgeNoopBackend) Call(context.Context, string, service.Request) (any, er func openBridgeForMCPTest(t *testing.T) *dotbridge.Service { t.Helper() - root := t.TempDir() + root := physicalTemp(t) vault := dotbridge.NewFakeVault() if err := dotbridge.EnsureStateKey(vault); err != nil { t.Fatal(err) @@ -96,3 +96,15 @@ func TestNewWithBridgeAddsTaskTools(t *testing.T) { t.Fatalf("tool count=%d want 17", len(list.Tools)) } } + +// physicalTemp resolves the test's temporary root: macOS temporary directories +// sit under the system /var symlink, which the stores deliberately reject as +// an ancestor. Production symlink rejection is unchanged. +func physicalTemp(t *testing.T) string { + t.Helper() + dir, err := filepath.EvalSymlinks(t.TempDir()) + if err != nil { + t.Fatal(err) + } + return dir +} From 0601dd089279d7b83cb841c59ee92e3253307bd6 Mon Sep 17 00:00:00 2001 From: Pyuyi <204892921+VeraPyuyi@users.noreply.github.com> Date: Wed, 21 Oct 2026 21:58:07 +0800 Subject: [PATCH 47/47] fix(queue): bound lock waits for race-slow transactions --- internal/dotbridge/lock_wait_test.go | 70 ++++++++++++++++++++++++++++ internal/dotqueue/store.go | 10 +++- 2 files changed, 79 insertions(+), 1 deletion(-) create mode 100644 internal/dotbridge/lock_wait_test.go diff --git a/internal/dotbridge/lock_wait_test.go b/internal/dotbridge/lock_wait_test.go new file mode 100644 index 0000000..e3aa880 --- /dev/null +++ b/internal/dotbridge/lock_wait_test.go @@ -0,0 +1,70 @@ +package dotbridge + +import ( + "net/http/httptest" + "sync" + "testing" + "time" + + "github.com/dotcontrol/dotcontrol/internal/dotqueue" +) + +// gateQueue blocks one List call while it owns the store lock, so a concurrent +// transaction contends with an in-flight read. +type gateQueue struct { + QueueAPI + once sync.Once + entered chan struct{} + release chan struct{} +} + +func (q *gateQueue) List(id string) ([]dotqueue.Job, error) { + q.once.Do(func() { + close(q.entered) + <-q.release + }) + return q.QueueAPI.List(id) +} + +type listOutcome struct { + code int + raw []byte +} + +// TestListAPIWaitsForConcurrentTransaction pins the store read path: a read +// that reaches the store while another transaction holds the lock waits for +// it rather than failing with dot_queue_unknown. CI reported that as an +// intermittent 400 from the macOS race run. +func TestListAPIWaitsForConcurrentTransaction(t *testing.T) { + s, token, _ := openTestBridge(t) + held := &gateQueue{QueueAPI: s.queue, entered: make(chan struct{}), release: make(chan struct{})} + s.queue = held + api := httptest.NewServer(s.Handler()) + defer api.Close() + + listed := make(chan listOutcome, 1) + go func() { + code, raw := listHTTPRequest(t, api, "GET", "/dot-bridge/v1/jobs", token, nil) + listed <- listOutcome{code, raw} + }() + <-held.entered + + // The same store, as the service would reach it, for the competing write. + store := held.QueueAPI.(QueueAPI) + writeErr := make(chan error, 1) + go func() { writeErr <- store.SetEnabled(true) }() + time.Sleep(150 * time.Millisecond) + close(held.release) + + if err := <-writeErr; err != nil { + t.Fatalf("concurrent write: %v", err) + } + select { + case got := <-listed: + if got.code != 200 { + t.Fatalf("list: %d %s", got.code, got.raw) + } + case <-time.After(15 * time.Second): + t.Fatal("list did not finish") + } +} diff --git a/internal/dotqueue/store.go b/internal/dotqueue/store.go index f9b60b4..a37858b 100644 --- a/internal/dotqueue/store.go +++ b/internal/dotqueue/store.go @@ -10,6 +10,11 @@ import ( "time" ) +// lockWait bounds how long a transaction waits for another to finish. It is +// generous enough for a race-enabled test writing a large result, while still +// failing closed rather than blocking forever. +const lockWait = 30 * time.Second + type Store struct { dir string key []byte @@ -416,7 +421,10 @@ func (s *Store) withLock(allowCreate bool, mutate bool, fn func(*persistenceStat if privatePath(lockPath, false) != nil { return errf("unknown") } - deadline := time.Now().Add(5 * time.Second) + // The wait stays bounded, but one transaction that rewrites a large state + // file can take several seconds under the race detector on a slow CI + // runner; five seconds made a concurrent read fail with unknown. + deadline := time.Now().Add(lockWait) for { ok, e := tryHandleLock(f) if e != nil {