From 8bd1369e16b3b55a76fe12bdb73fc47d9c4e8f75 Mon Sep 17 00:00:00 2001 From: ranxi2001 Date: Mon, 16 Nov 2026 17:24:59 +0800 Subject: [PATCH 1/9] =?UTF-8?q?fix:=20=E5=81=9C=E8=B0=83=E8=B4=A6=E5=8F=B7?= =?UTF-8?q?=E5=90=8E=E7=9A=84=E6=9C=AA=E5=8F=91=E9=80=81=E5=B7=A5=E5=85=B7?= =?UTF-8?q?=E5=A2=9E=E9=87=8F=E6=81=A2=E5=A4=8D=E6=9C=AC=E8=BF=9E=E6=8E=A5?= =?UTF-8?q?=E5=8F=AF=E8=AF=BB=E5=8E=86=E5=8F=B2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/ws-unsent-delta.yml | 77 +++++ .../handler/openai_ws_unsent_delta_test.go | 194 +++++++++++ .../service/openai_ws_agent_recovery.go | 300 ++++++++++++++++++ .../service/openai_ws_forwarder_ingress.go | 5 + .../openai_ws_readable_continuation.go | 6 + 5 files changed, 582 insertions(+) create mode 100644 .github/workflows/ws-unsent-delta.yml create mode 100644 backend/internal/handler/openai_ws_unsent_delta_test.go diff --git a/.github/workflows/ws-unsent-delta.yml b/.github/workflows/ws-unsent-delta.yml new file mode 100644 index 000000000000..f74b5871a0fc --- /dev/null +++ b/.github/workflows/ws-unsent-delta.yml @@ -0,0 +1,77 @@ +name: WS unsent delta takeover +on: + pull_request: + paths: + - 'backend/internal/service/openai_ws_agent_recovery.go' + - 'backend/internal/service/openai_ws_forwarder_ingress.go' + - 'backend/internal/service/openai_ws_readable_continuation.go' + - 'backend/internal/handler/openai_ws_unsent_delta_test.go' + - '.github/workflows/ws-unsent-delta.yml' +permissions: + contents: read +jobs: + takeover: + runs-on: ubuntu-24.04 + timeout-minutes: 25 + strategy: + fail-fast: false + matrix: + phase: [RED, GREEN] + steps: + - uses: actions/checkout@v6 + with: + ref: ${{ github.event.pull_request.head.sha }} + fetch-depth: 0 + persist-credentials: false + - uses: actions/setup-go@v6 + with: + go-version-file: backend/go.mod + cache-dependency-path: backend/go.sum + - name: Redis fixture dependency + run: | + if ! command -v redis-server >/dev/null; then + sudo apt-get update + sudo apt-get install -y --no-install-recommends redis-server + fi + - name: Exact baseline implementation for RED + if: matrix.phase == 'RED' + run: | + for target in backend/internal/service/openai_ws_agent_recovery.go backend/internal/service/openai_ws_forwarder_ingress.go backend/internal/service/openai_ws_readable_continuation.go; do + git show "2927209f5492d8cb690b83a1964b2f093c19ad7a:$target" > "$target" + test "$(git hash-object "$target")" = "$(git rev-parse "2927209f5492d8cb690b83a1964b2f093c19ad7a:$target")" + done + - name: Same socket, account pause, result delta and next turn + env: + PHASE: ${{ matrix.phase }} + shell: bash + run: | + set -euo pipefail + set +e + (cd backend && go test -race -json -tags=unit,unsentdelta ./internal/handler -run '^TestOpenAIWSUnsentDelta(Takeover|Fences)$' -count=1 -timeout=5m) > "$RUNNER_TEMP/unsent-delta.jsonl" 2>&1 + result=$? + set -e + cat "$RUNNER_TEMP/unsent-delta.jsonl" + python3 .github/scripts/assert-readable-continuation.py --clean-log "$RUNNER_TEMP/unsent-delta.jsonl" + python3 - "$PHASE" "$result" "$RUNNER_TEMP/unsent-delta.jsonl" <<'PY' + import json,sys + phase,status,path=sys.argv[1],int(sys.argv[2]),sys.argv[3] + rows=[json.loads(l) for l in open(path) if l.startswith('{')] + outcomes={r.get('Test'):r['Action'] for r in rows if r.get('Action') in ('pass','fail','skip') and r.get('Test')} + modes=('http_bridge','ctx_pool','shared','dedicated','passthrough') + expected={f'TestOpenAIWSUnsentDeltaTakeover/{m}/turn_{n}' for m in modes for n in (2,53)} + fences={f'TestOpenAIWSUnsentDeltaFences/{m}/{f}' for m in ('http_bridge','ctx_pool','passthrough') for f in ('unknown_pointer','unknown_call','terminal_excludes','beta_off')} + assert all(outcomes.get(n)=='pass' for n in fences),outcomes + if phase=='RED': + assert status!=0 and all(outcomes.get(n)=='fail' for n in expected),outcomes + for name in expected: + assert 'UNSENT_DELTA_NOT_RESUMED' in ''.join(r.get('Output','') for r in rows if r.get('Test')==name),name + else: + assert status==0 and all(outcomes.get(n)=='pass' for n in expected),outcomes + assert not any(a in ('fail','skip') for a in outcomes.values()),outcomes + PY + - uses: actions/upload-artifact@v7 + if: always() + with: + name: unsent-delta-${{ matrix.phase }}-${{ github.event.pull_request.head.sha }} + path: ${{ runner.temp }}/unsent-delta.jsonl + if-no-files-found: error diff --git a/backend/internal/handler/openai_ws_unsent_delta_test.go b/backend/internal/handler/openai_ws_unsent_delta_test.go new file mode 100644 index 000000000000..2a35cc4d83e9 --- /dev/null +++ b/backend/internal/handler/openai_ws_unsent_delta_test.go @@ -0,0 +1,194 @@ +//go:build unit && unsentdelta + +package handler + +import ( + "context" + "encoding/json" + "fmt" + "github.com/Wei-Shaw/sub2api/internal/config" + "github.com/Wei-Shaw/sub2api/internal/repository" + "github.com/Wei-Shaw/sub2api/internal/service" + coderws "github.com/coder/websocket" + "github.com/stretchr/testify/require" + "github.com/tidwall/gjson" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + "time" +) + +func TestOpenAIWSUnsentDeltaTakeover(t *testing.T) { + for _, mode := range readableWSModes { + for _, turn := range []int{2, 53} { + t.Run(fmt.Sprintf("%s/turn_%d", mode, turn), func(t *testing.T) { runUnsentDelta(t, mode, turn, "") }) + } + } +} +func TestOpenAIWSUnsentDeltaFences(t *testing.T) { + for _, mode := range []string{service.OpenAIWSIngressModeHTTPBridge, service.OpenAIWSIngressModeCtxPool, service.OpenAIWSIngressModePassthrough} { + for _, fence := range []string{"unknown_pointer", "unknown_call", "terminal_excludes", "beta_off"} { + t.Run(mode+"/"+fence, func(t *testing.T) { runUnsentDelta(t, mode, 2, fence) }) + } + } +} +func runUnsentDelta(t *testing.T, mode string, switchTurn int, fence string) { + t.Helper() + var mu sync.Mutex + var frames [][]byte + var owners []string + response := func(body []byte, auth string) []byte { + mu.Lock() + defer mu.Unlock() + frames = append(frames, append([]byte(nil), body...)) + owners = append(owners, auth) + n := len(frames) + output := fmt.Sprintf(`[{"type":"message","role":"assistant","content":[{"type":"output_text","text":"fixture history %d"}]}]`, n) + if n == switchTurn-1 { + output = `[{"type":"function_call","call_id":"call_takeover_once","name":"fixture","arguments":"{\"n\":9007199254740993}","status":"completed"}]` + } + if n == switchTurn-1 && fence == "terminal_excludes" { + output = `[{"type":"message","role":"assistant","content":[{"type":"output_text","text":"no tool delivered"}]}]` + } + return []byte(fmt.Sprintf(`{"type":"response.completed","response":{"id":"resp_delta_%d","model":"gpt-5.1","status":"completed","output":%s,"usage":{"input_tokens":5,"output_tokens":3}}}`, n, output)) + } + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if !strings.EqualFold(r.Header.Get("Upgrade"), "websocket") { + body, e := io.ReadAll(r.Body) + if e != nil { + t.Error(e) + return + } + w.Header().Set("Content-Type", "text/event-stream") + fmt.Fprintf(w, "data: %s\n\n", response(body, r.Header.Get("Authorization"))) + return + } + conn, e := coderws.Accept(w, r, nil) + if e != nil { + t.Error(e) + return + } + defer conn.CloseNow() + for { + _, body, e := conn.Read(r.Context()) + if e != nil { + return + } + if conn.Write(r.Context(), coderws.MessageText, response(body, r.Header.Get("Authorization"))) != nil { + return + } + } + })) + defer upstream.Close() + clients := issue805Redis(t) + headers := http.Header{"Session_id": {"fixture-delta-session"}, "Thread-Id": {"fixture-delta-thread"}} + var accounts *readableWSAccounts + h := newOpenAIWSPassthroughHandlerHarnessWithKey(t, upstream.URL, headers, + map[string]string{service.SettingKeyCyberSessionBlockEnabled: "false", service.SettingKeyOpenAI409OmitEncryptedHistoryEnabled: fmt.Sprint(fence != "beta_off")}, + repository.NewGatewayCache(clients()), func(k *service.APIKey) { k.AllowLongContext = true }, + func(a *service.Account, cfg *config.Config, _ *service.HTTPUpstream, repo *service.AccountRepository) { + a.ID = 1 + a.Priority = 0 + a.Credentials["api_key"] = "fixture-A" + a.Extra["openai_apikey_responses_websockets_v2_mode"] = mode + a.Extra["codex_fingerprint_mode"] = "off" + cfg.Gateway.MaxAccountSwitches = 2 + cfg.Gateway.Scheduling.OpenAITurnAdmissionCacheTTLSeconds = 0 + cfg.Gateway.OpenAIWS.IngressInterTurnIdleTimeoutSeconds = 20 + b := *a + b.ID = 2 + b.Priority = 1 + b.Credentials = map[string]any{"api_key": "fixture-B", "base_url": upstream.URL} + accounts = &readableWSAccounts{issue600SwitchRepo: &issue600SwitchRepo{&openAIWSFailoverHandlerAccountRepoStub{accounts: []service.Account{*a, b}}}} + *repo = accounts + }) + defer issue600Finish(t, h, headers) + create := func(input, previous string) string { + b := map[string]any{"type": "response.create", "model": "gpt-5.1", "store": true, "input": json.RawMessage(input)} + if previous != "" { + b["previous_response_id"] = previous + } else { + b["tools"] = []any{map[string]any{"type": "function", "name": "fixture", "parameters": map[string]any{"type": "object", "properties": map[string]any{}}}} + } + raw, _ := json.Marshal(b) + return string(raw) + } + issue715WSSend(t, h.clientConn, create(`[{"role":"user","content":"Perform fixture exactly once and retain the task"}]`, "")) + executions := 0 + consume := func(raw []byte) { + for _, item := range gjson.GetBytes(raw, "response.output").Array() { + if item.Get("type").String() == "function_call" { + require.Equal(t, "call_takeover_once", item.Get("call_id").String()) + executions++ + } + } + } + first := issue715WSRead(t, h.clientConn) + consume(first) + require.Equal(t, "response.completed", gjson.GetBytes(first, "type").String()) + for n := 2; n < switchTurn; n++ { + issue715WSSend(t, h.clientConn, create(fmt.Sprintf(`[{"role":"user","content":"fixture next %d"}]`, n), fmt.Sprintf("resp_delta_%d", n-1))) + round := issue715WSRead(t, h.clientConn) + consume(round) + require.Equal(t, "response.completed", gjson.GetBytes(round, "type").String()) + } + accounts.disable(1) + previous := fmt.Sprintf("resp_delta_%d", switchTurn-1) + if fence == "unknown_pointer" { + previous = "resp_foreign" + } + call := "call_takeover_once" + if fence == "unknown_call" { + call = "call_fabricated" + } + issue715WSSend(t, h.clientConn, create(fmt.Sprintf(`[{"type":"function_call_output","call_id":%q,"output":"fixture applied exactly once"}]`, call), previous)) + if fence != "" { + readCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _, raw, e := h.clientConn.Read(readCtx) + mu.Lock() + n := len(frames) + mu.Unlock() + require.Equal(t, switchTurn-1, n, "unsafe delta cannot emit another generation") + require.True(t, e != nil || gjson.GetBytes(raw, "type").String() == "error") + return + } + result := issue715WSRead(t, h.clientConn) + consume(result) + if gjson.GetBytes(result, "type").String() != "response.completed" { + logReadableWSRecoveryDetails(t, h) + } + require.Equal(t, "response.completed", gjson.GetBytes(result, "type").String(), "UNSENT_DELTA_NOT_RESUMED: %s", result) + mu.Lock() + sent := append([][]byte(nil), frames...) + selected := append([]string(nil), owners...) + mu.Unlock() + require.Len(t, sent, switchTurn) + require.Equal(t, "Bearer fixture-B", selected[switchTurn-1]) + resumed := sent[switchTurn-1] + require.NotContains(t, string(resumed), "resp_delta_", "old provider pointer must not reach B") + require.Contains(t, string(resumed), "Perform fixture exactly once") + require.Contains(t, string(resumed), "fixture applied exactly once") + require.Contains(t, string(resumed), "9007199254740993") + require.True(t, gjson.GetBytes(resumed, "tools").IsArray(), "effective tool declarations must survive takeover") + for _, item := range gjson.GetBytes(resumed, "input").Array() { + require.NotContains(t, item.Get("type").String(), "call", "completed tools must be inert history") + } + if switchTurn == 53 { + require.Contains(t, string(resumed), "fixture history 1") + require.Contains(t, string(resumed), "fixture next 52") + } + require.Equal(t, 1, executions) + issue715WSSend(t, h.clientConn, create(`[{"role":"user","content":"fixture final followup"}]`, fmt.Sprintf("resp_delta_%d", switchTurn))) + round := issue715WSRead(t, h.clientConn) + consume(round) + require.Equal(t, "response.completed", gjson.GetBytes(round, "type").String()) + mu.Lock() + defer mu.Unlock() + require.Len(t, frames, switchTurn+1) + require.Equal(t, "Bearer fixture-B", owners[switchTurn]) + require.Equal(t, 1, executions) +} diff --git a/backend/internal/service/openai_ws_agent_recovery.go b/backend/internal/service/openai_ws_agent_recovery.go index a68528173054..451ae252af66 100644 --- a/backend/internal/service/openai_ws_agent_recovery.go +++ b/backend/internal/service/openai_ws_agent_recovery.go @@ -4,17 +4,20 @@ import ( "bytes" "context" "encoding/json" + "errors" "io" "net/http" "net/url" "sync" "sync/atomic" "time" + "unicode/utf8" "github.com/Wei-Shaw/sub2api/internal/pkg/logger" coderws "github.com/coder/websocket" "github.com/gin-gonic/gin" "github.com/tidwall/gjson" + "github.com/tidwall/sjson" "go.uber.org/zap" ) @@ -63,6 +66,8 @@ type openAIWSAgentRecovery struct { completedBridge openAIWSBridgeResponseAnchor deliveredTools map[string]openAIWSDeliveredTool pendingTools map[string]openAIWSDeliveredTool + localHistory openAIWSLocalHistory + historyReserved int64 } func newOpenAIWSAgentRecovery(s *OpenAIGatewayService, ctx context.Context, c *gin.Context) *openAIWSAgentRecovery { @@ -89,6 +94,7 @@ func (r *openAIWSAgentRecovery) abort() { } r.mu.Lock() defer r.mu.Unlock() + defer func() { r.localHistory = openAIWSLocalHistory{}; r.accountLocalHistory() }() if r.finish != nil { r.finish(false) r.finish = nil @@ -407,6 +413,7 @@ func (r *openAIWSAgentRecovery) begin(account *Account, source, payload []byte, return err } r.finish = finish + r.stageLocalHistory(source) if r.request.owner != nil || len(r.request.items) > 0 || r.request.omittedHistory || (r.service.settingService != nil && r.service.settingService.IsOpenAI409OmitEncryptedHistoryEnabled(ctx)) { // In automatic continuation mode, even a plaintext initial generation @@ -460,6 +467,15 @@ func (r *openAIWSAgentRecovery) write(payload []byte, write func() error) error r.finish = nil return err } + if r.localHistory.pending != nil { + r.localHistory.outputBytes += len(payload) + if r.localHistory.outputBytes > openAIWSLocalHistoryLimit { + r.localHistory.pending = nil + } else { + r.localHistory.output.AddEvent(event, string(payload)) + r.accountLocalHistory() + } + } toolItems := gjson.GetBytes(payload, "response.output").Array() if event == "response.output_item.done" { toolItems = append(toolItems, gjson.GetBytes(payload, "item")) @@ -555,6 +571,7 @@ func (r *openAIWSAgentRecovery) write(payload []byte, write func() error) error r.known[request.itemKey(item)] = true } } + r.commitLocalHistory(completed) r.finish = nil } return nil @@ -598,3 +615,286 @@ func openAIWSWireRefusal(ctx context.Context, stage string, source, payload []by zap.Bool("controls_changed", openAI409ControlsDigest(source) != openAI409ControlsDigest(payload))) return openAIRecoveryDiagnosticError("wire_changed", stage, openAIAgentRecoveryFromContext(ctx)) } + +// Bounded, manager-local plaintext history for an explicitly unsent next turn. +// It expires with the socket/forwarding attempt, never enters Redis or logs. +const openAIWSLocalHistoryLimit = 8 << 20 + +var openAIWSLocalHistoryBytes atomic.Int64 + +type openAIWSLocalHistory struct { + outputBytes int + tools json.RawMessage + instructions json.RawMessage + committed []json.RawMessage + pending []json.RawMessage + output openAIWSToolCallReplayCollector + anchor openAIWSBridgeResponseAnchor + clientModel string + ready bool +} + +type openAIWSRestoredUnsentError struct{ *openAIWSUnsentTurnError } + +func (e *openAIWSRestoredUnsentError) Unwrap() error { return e.openAIWSUnsentTurnError } +func (e *openAIWSRestoredUnsentError) RecoveredWSContext(ctx context.Context) context.Context { + return WithRecoveredContextPrivacy(ctx) +} + +func openAIWSPlainItems(body []byte) ([]json.RawMessage, bool) { + if len(body) > openAIWSLocalHistoryLimit { + return nil, false + } + if !utf8.Valid(body) || !openAIEncryptedJSONUnambiguous(body) { + return nil, false + } + input := gjson.GetBytes(body, "input") + var items []json.RawMessage + if input.Type == gjson.String { + raw, _ := json.Marshal(map[string]any{"type": "message", "role": "user", "content": input.String()}) + items = []json.RawMessage{raw} + } else if input.IsArray() { + for _, item := range input.Array() { + items = append(items, json.RawMessage(item.Raw)) + } + } else { + return nil, false + } + size := 0 + for _, raw := range items { + size += len(raw) + if size > openAIWSLocalHistoryLimit || !openAIWSPlainItem(raw) { + return nil, false + } + } + return items, true +} +func openAIWSPlainItem(raw []byte) bool { + item := gjson.ParseBytes(raw) + if !item.IsObject() || item.Get("encrypted_content").Exists() { + return false + } + switch item.Get("type").String() { + case "", "message": + if role := item.Get("role").String(); role != "user" && role != "assistant" && role != "system" && role != "developer" { + return false + } + return openAIReadableShadowTextOnly(item.Get("content")) + case "function_call", "custom_tool_call": + _, ok := openAIToolFingerprint(item) + return ok + case "function_call_output", "custom_tool_call_output": + return item.Get("call_id").Type == gjson.String && item.Get("call_id").String() != "" && openAIReadableShadowTextOnly(item.Get("output")) + default: + return false + } +} +func openAIWSHistoryCopy(items []json.RawMessage) ([]json.RawMessage, bool) { + size := 0 + for _, v := range items { + size += len(v) + if size > openAIWSLocalHistoryLimit { + return nil, false + } + } + out := make([]json.RawMessage, len(items)) + for i, v := range items { + out[i] = append(json.RawMessage(nil), v...) + } + return out, true +} +func (r *openAIWSAgentRecovery) stageLocalHistory(source []byte) { + defer r.accountLocalHistory() + r.localHistory.pending = nil + r.localHistory.outputBytes = 0 + r.localHistory.output = openAIWSToolCallReplayCollector{} + if r.service.settingService == nil || !r.service.settingService.IsOpenAI409OmitEncryptedHistoryEnabled(r.ctx) { + r.localHistory = openAIWSLocalHistory{} + return + } + items, ok := openAIWSPlainItems(source) + if !ok || gjson.GetBytes(source, "conversation").Exists() { + r.localHistory.ready = false + return + } + previous := gjson.GetBytes(source, "previous_response_id").String() + if previous != "" { + anchor := r.localHistory.anchor + if !r.localHistory.ready || previous != anchor.response || r.request.keyID != anchor.key || r.request.userID != anchor.user || r.request.groupID != anchor.group || r.request.session != anchor.session || r.request.model != anchor.model { + r.localHistory.ready = false + return + } + // Accept either a delta or the exact complete prefix; never drop existing messages. + if !openAIWSRawItemsHasPrefix(items, r.localHistory.committed) { + items = combineOpenAIWSReplayItems(r.localHistory.committed, items) + } + } + if previous == "" { + r.localHistory.tools = nil + r.localHistory.instructions = nil + } + for field, dst := range map[string]*json.RawMessage{"tools": &r.localHistory.tools, "instructions": &r.localHistory.instructions} { + if value := gjson.GetBytes(source, field); value.Exists() { + *dst = append(json.RawMessage(nil), value.Raw...) + } + } + if len(r.localHistory.tools)+len(r.localHistory.instructions) > openAIWSLocalHistoryLimit { + r.localHistory = openAIWSLocalHistory{} + return + } + r.localHistory.pending, ok = openAIWSHistoryCopy(items) + r.localHistory.ready = false + if !ok { + r.localHistory.pending = nil + } + if model := gjson.GetBytes(source, "model").String(); model != "" { + r.localHistory.clientModel = model + } +} +func (r *openAIWSAgentRecovery) commitLocalHistory(completed bool) { + defer r.accountLocalHistory() + h := &r.localHistory + if !completed || h.pending == nil { + h.ready = false + h.committed = nil + return + } + output := h.output.AllItems() + if h.output.finalItems != nil { + output = h.output.finalItems + } + for _, item := range output { + if !openAIWSPlainItem(item) { + h.ready = false + h.committed = nil + return + } + } + all := combineOpenAIWSReplayItems(h.pending, output) + var ok bool + h.committed, ok = openAIWSHistoryCopy(all) + h.ready = ok + if ok { + h.anchor = r.completedBridge + } + h.pending = nil + h.output = openAIWSToolCallReplayCollector{} +} + +// Called at the outer proxy return boundary, after transport goroutines stop. +// Only the typed not-written admission error grants permission to attach prior +// delivered context. Every current auth/moderation/long-context gate runs again. +func (r *openAIWSAgentRecovery) restoreUnsentHistory(err error) error { + if r == nil { + return err + } + var unsent *openAIWSUnsentTurnError + var admission *OpenAITurnAdmissionError + if !errors.As(err, &unsent) || unsent.keepAccount || !errors.As(err, &admission) { + return err + } + r.mu.Lock() + defer r.mu.Unlock() + h := &r.localHistory + if !h.ready || r.finish != nil || r.service.settingService == nil || !r.service.settingService.IsOpenAI409OmitEncryptedHistoryEnabled(r.ctx) { + return err + } + source := unsent.body + if len(source) > openAIWSLocalHistoryLimit || gjson.GetBytes(source, "previous_response_id").String() != h.anchor.response || gjson.GetBytes(source, "conversation").Exists() { + return err + } + if unsent.model != "" && unsent.model != h.clientModel { + return err + } + scope := openAIAgentRecoveryFromContext(r.service.WithOpenAIAgentMessageRecovery(r.ctx, r.c, source, r.session, h.anchor.model)) + if scope.userID != h.anchor.user || scope.keyID != h.anchor.key || scope.groupID != h.anchor.group || scope.session != h.anchor.session || scope.lookupErr != nil || scope.conflict || scope.disallowLearning { + return err + } + delta, ok := openAIWSPlainItems(source) + if !ok { + return err + } + all := delta + if !openAIWSRawItemsHasPrefix(delta, h.committed) { + all = combineOpenAIWSReplayItems(h.committed, delta) + } + all, ok = openAIWSHistoryCopy(all) + if !ok { + return err + } + expanded, e := setOpenAIWSPayloadInputSequence(source, all, true) + if e != nil { + return err + } + if _, why := inspectOpenAIWSCompletedTools(expanded); why != "" { + return err + } + // Every executed historical call must be one actually delivered on this + // manager with a successful terminal verdict, including encrypted anchors. + for _, item := range gjson.GetBytes(expanded, "input").Array() { + kind := item.Get("type").String() + if kind != "function_call" && kind != "custom_tool_call" { + continue + } + fp, valid := openAIToolFingerprint(item) + proof, found := r.deliveredTools[h.anchor.model+"\x00"+item.Get("call_id").String()] + if !valid || !found || proof.conflicted || proof.fingerprint != fp || proof.accountID != h.anchor.account || proof.config != h.anchor.config { + return err + } + } + for field, value := range map[string]json.RawMessage{"tools": h.tools, "instructions": h.instructions} { + if !gjson.GetBytes(expanded, field).Exists() && len(value) > 0 { + expanded, e = sjson.SetRawBytes(expanded, field, value) + if e != nil { + return err + } + } + } + if !openAI409HasReadableIntent(expanded) { + return err + } + // Completed calls become inert history, so a replacement provider cannot + // execute the original tool call again. Keep arguments/results as exact JSON. + expanded, e = projectOpenAIWSCompletedTools(expanded) + if e != nil { + return err + } + expanded, e = sjson.DeleteBytes(expanded, "previous_response_id") + if e != nil || len(expanded) > openAIWSLocalHistoryLimit { + return err + } + copy := *unsent + copy.body = expanded + h.ready = false // proof consumed once; no reuse after any downstream failure + return &openAIWSRestoredUnsentError{©} +} + +// Budget retained context across all sockets; exhaustion only disables this +// optional restoration path. It never rejects normal forwarding or drops bytes. +func (r *openAIWSAgentRecovery) accountLocalHistory() { + h := &r.localHistory + wanted := int64(len(h.tools) + len(h.instructions) + 3*h.outputBytes) + for _, set := range [][]json.RawMessage{h.pending, h.committed} { + for _, v := range set { + wanted += int64(len(v)) + } + } + delta := wanted - r.historyReserved + if delta > 0 { + for { + old := openAIWSLocalHistoryBytes.Load() + if old+delta > 128<<20 { + r.localHistory = openAIWSLocalHistory{} + openAIWSLocalHistoryBytes.Add(-r.historyReserved) + r.historyReserved = 0 + return + } + if openAIWSLocalHistoryBytes.CompareAndSwap(old, old+delta) { + break + } + } + } else { + openAIWSLocalHistoryBytes.Add(delta) + } + r.historyReserved = wanted +} diff --git a/backend/internal/service/openai_ws_forwarder_ingress.go b/backend/internal/service/openai_ws_forwarder_ingress.go index 24dc2661ca5e..32c94fe8e8ba 100644 --- a/backend/internal/service/openai_ws_forwarder_ingress.go +++ b/backend/internal/service/openai_ws_forwarder_ingress.go @@ -589,6 +589,11 @@ func (s *OpenAIGatewayService) ProxyResponsesWebSocketFromClient( agentRecovery := newOpenAIWSAgentRecovery(s, ctx, c) ctx = context.WithValue(ctx, openAIWSAgentRecoveryKey{}, agentRecovery) defer agentRecovery.abort() + defer func() { + if recovery, ok := any(agentRecovery).(interface{ restoreUnsentHistory(error) error }); ok { + returnErr = recovery.restoreUnsentHistory(returnErr) + } + }() var meterCleanup func() var meterErr error ctx, hooks, meterCleanup, meterErr = s.wrapRegionalWSTurns(ctx, c, account, firstClientMessage, hooks) diff --git a/backend/internal/service/openai_ws_readable_continuation.go b/backend/internal/service/openai_ws_readable_continuation.go index 355fb825f7bd..1a966587f726 100644 --- a/backend/internal/service/openai_ws_readable_continuation.go +++ b/backend/internal/service/openai_ws_readable_continuation.go @@ -103,6 +103,12 @@ type openAIWSUnsentOwnerProofKey struct{} // Carry the route check's observations through the handler's auth boundary. // Rebuild consumes this current-turn evidence once; it cannot leak to later turns. func WithOpenAIWSUnsentOwnerProof(ctx context.Context, err error) context.Context { + var recovered interface { + RecoveredWSContext(context.Context) context.Context + } + if errors.As(err, &recovered) { + ctx = recovered.RecoveredWSContext(ctx) + } var unsent *openAIWSUnsentTurnError if errors.As(err, &unsent) && unsent.completedSource != nil { return context.WithValue(ctx, openAIWSUnsentOwnerProofKey{}, unsent.completedSource) From 9df51d9bc4ad60f0824cdd00d561357efb494447 Mon Sep 17 00:00:00 2001 From: ranxi2001 Date: Mon, 16 Nov 2026 17:28:26 +0800 Subject: [PATCH 2/9] =?UTF-8?q?fix:=20=E5=AF=B9=E9=BD=90=E8=BE=93=E5=87=BA?= =?UTF-8?q?=E8=A7=82=E5=AF=9F=E6=8E=A5=E5=8F=A3=E5=B9=B6=E8=A1=A5=E4=BA=A4?= =?UTF-8?q?=E4=BB=98=E4=BD=9C=E7=94=A8=E5=9F=9F=E4=B8=8E=E5=86=85=E5=AD=98?= =?UTF-8?q?=E5=9B=9E=E5=BD=92?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/ws-unsent-delta.yml | 9 ++- .../service/openai_ws_agent_recovery.go | 6 +- .../service/openai_ws_unsent_delta_test.go | 81 +++++++++++++++++++ 3 files changed, 93 insertions(+), 3 deletions(-) create mode 100644 backend/internal/service/openai_ws_unsent_delta_test.go diff --git a/.github/workflows/ws-unsent-delta.yml b/.github/workflows/ws-unsent-delta.yml index f74b5871a0fc..6bd80e350cb1 100644 --- a/.github/workflows/ws-unsent-delta.yml +++ b/.github/workflows/ws-unsent-delta.yml @@ -6,6 +6,7 @@ on: - 'backend/internal/service/openai_ws_forwarder_ingress.go' - 'backend/internal/service/openai_ws_readable_continuation.go' - 'backend/internal/handler/openai_ws_unsent_delta_test.go' + - 'backend/internal/service/openai_ws_unsent_delta_test.go' - '.github/workflows/ws-unsent-delta.yml' permissions: contents: read @@ -69,9 +70,15 @@ jobs: assert status==0 and all(outcomes.get(n)=='pass' for n in expected),outcomes assert not any(a in ('fail','skip') for a in outcomes.values()),outcomes PY + - name: GREEN delivery scope and memory release + if: matrix.phase == 'GREEN' + shell: bash + run: | + set -euo pipefail + (cd backend && go test -race -json -tags=unit,unsentdelta ./internal/service -run '^TestOpenAIWSUnsentDeltaDeliveryScope$' -count=1 -timeout=2m) | tee "$RUNNER_TEMP/unsent-delta-guards.jsonl" - uses: actions/upload-artifact@v7 if: always() with: name: unsent-delta-${{ matrix.phase }}-${{ github.event.pull_request.head.sha }} - path: ${{ runner.temp }}/unsent-delta.jsonl + path: ${{ runner.temp }}/unsent-delta*.jsonl if-no-files-found: error diff --git a/backend/internal/service/openai_ws_agent_recovery.go b/backend/internal/service/openai_ws_agent_recovery.go index 451ae252af66..e4ba87745d10 100644 --- a/backend/internal/service/openai_ws_agent_recovery.go +++ b/backend/internal/service/openai_ws_agent_recovery.go @@ -470,9 +470,10 @@ func (r *openAIWSAgentRecovery) write(payload []byte, write func() error) error if r.localHistory.pending != nil { r.localHistory.outputBytes += len(payload) if r.localHistory.outputBytes > openAIWSLocalHistoryLimit { - r.localHistory.pending = nil + r.localHistory = openAIWSLocalHistory{} + r.accountLocalHistory() } else { - r.localHistory.output.AddEvent(event, string(payload)) + r.localHistory.output.AddEvent(event, payload) r.accountLocalHistory() } } @@ -778,6 +779,7 @@ func (r *openAIWSAgentRecovery) commitLocalHistory(completed bool) { h.anchor = r.completedBridge } h.pending = nil + h.outputBytes = 0 h.output = openAIWSToolCallReplayCollector{} } diff --git a/backend/internal/service/openai_ws_unsent_delta_test.go b/backend/internal/service/openai_ws_unsent_delta_test.go new file mode 100644 index 000000000000..429146d75e20 --- /dev/null +++ b/backend/internal/service/openai_ws_unsent_delta_test.go @@ -0,0 +1,81 @@ +//go:build unsentdelta + +package service + +import ( + "context" + "errors" + "github.com/Wei-Shaw/sub2api/internal/config" + "github.com/stretchr/testify/require" + "strings" + "testing" +) + +func TestOpenAIWSUnsentDeltaDeliveryScope(t *testing.T) { + for _, mutation := range []string{"valid", "pending", "failed_write", "terminal_excludes", "unknown_call", "changed_call", "wrong_pointer", "wrong_key", "wrong_model", "sent_error", "oversize"} { + t.Run(mutation, func(t *testing.T) { + before := openAIWSLocalHistoryBytes.Load() + svc := &OpenAIGatewayService{cfg: &config.Config{}, settingService: NewSettingService(&codexPolicyMigrationRepoStub{values: map[string]string{SettingKeyOpenAI409OmitEncryptedHistoryEnabled: "true"}}, nil)} + manager, c, a := issue715WSManager(t, svc, context.Background()) + SetOpenAIWSRecoveryPrincipal(c, 715, 715) + defer func() { + manager.abort() + require.Equal(t, before, openAIWSLocalHistoryBytes.Load(), "socket teardown must release retained history") + }() + source := `{"model":"gpt-6-astra","input":[{"role":"user","content":"fixture original task"}]}` + require.NoError(t, issue715WSBegin(t, manager, a, source, "ctx_pool")) + call := `{"type":"function_call","call_id":"call_fixture_delta","name":"fixture","arguments":"{}","status":"completed"}` + if mutation == "pending" { + require.NoError(t, manager.write([]byte(`{"type":"response.output_item.done","item":`+call+`}`), func() error { return nil })) + } else { + output := call + if mutation == "terminal_excludes" { + output = `{"type":"message","role":"assistant","content":"fixture complete without tool"}` + } + err := manager.write([]byte(`{"type":"response.completed","response":{"id":"resp_fixture_delta","status":"completed","output":[`+output+`]}}`), func() error { + if mutation == "failed_write" { + return errors.New("fixture write failed") + } + return nil + }) + if mutation == "failed_write" { + require.Error(t, err) + } else { + require.NoError(t, err) + } + } + current := `{"type":"response.create","model":"gpt-6-astra","previous_response_id":"resp_fixture_delta","input":[{"type":"function_call_output","call_id":"call_fixture_delta","output":"fixture executed once"}]}` + model := "gpt-6-astra" + switch mutation { + case "unknown_call": + current = strings.Replace(current, "call_fixture_delta", "call_unknown", 1) + case "changed_call": + manager.deliveredTools[model+"\x00call_fixture_delta"] = openAIWSDeliveredTool{fingerprint: "wrong", accountID: a.ID} + case "wrong_pointer": + current = strings.Replace(current, "resp_fixture_delta", "resp_foreign", 1) + case "wrong_key": + c.Set("api_key", &APIKey{ID: 999, GroupID: int64Ptr(715)}) + case "wrong_model": + model = "gpt-6-sol" + case "oversize": + current = strings.Replace(current, "fixture executed once", strings.Repeat("x", openAIWSLocalHistoryLimit+1), 1) + } + cause := openAIWSUnsentTurn(denyOpenAITurn("account_ineligible"), []byte(current), model, 2) + if mutation == "sent_error" { + cause = errors.New("fixture unknown upstream delivery") + } + got := manager.restoreUnsentHistory(cause) + var restored *openAIWSRestoredUnsentError + if mutation == "valid" { + require.ErrorAs(t, got, &restored) + require.Contains(t, string(restored.body), "fixture original task") + require.Contains(t, string(restored.body), "fixture executed once") + require.NotContains(t, string(restored.body), `"type":"function_call"`) + require.NotContains(t, string(restored.body), "previous_response_id") + require.Same(t, cause, manager.restoreUnsentHistory(cause), "proof is consumed once") + } else { + require.Same(t, cause, got, "no evidence may be fabricated from failed delivery or foreign scope") + } + }) + } +} From c16ab41dd27b0bdcd1d07c1b0faa635512d250a0 Mon Sep 17 00:00:00 2001 From: ranxi2001 Date: Mon, 16 Nov 2026 17:29:07 +0800 Subject: [PATCH 3/9] =?UTF-8?q?fix:=20=E9=99=90=E5=AE=9A=E6=9C=AC=E8=BF=9E?= =?UTF-8?q?=E6=8E=A5=E5=8E=86=E5=8F=B2=E6=81=A2=E5=A4=8D=E4=BD=BF=E7=94=A8?= =?UTF-8?q?=E5=B7=B2=E8=AE=A4=E8=AF=81=E7=9A=84=E7=94=A8=E6=88=B7=E4=B8=8E?= =?UTF-8?q?=20Key?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/internal/service/openai_ws_agent_recovery.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/internal/service/openai_ws_agent_recovery.go b/backend/internal/service/openai_ws_agent_recovery.go index e4ba87745d10..a0d4577cb37b 100644 --- a/backend/internal/service/openai_ws_agent_recovery.go +++ b/backend/internal/service/openai_ws_agent_recovery.go @@ -809,7 +809,7 @@ func (r *openAIWSAgentRecovery) restoreUnsentHistory(err error) error { return err } scope := openAIAgentRecoveryFromContext(r.service.WithOpenAIAgentMessageRecovery(r.ctx, r.c, source, r.session, h.anchor.model)) - if scope.userID != h.anchor.user || scope.keyID != h.anchor.key || scope.groupID != h.anchor.group || scope.session != h.anchor.session || scope.lookupErr != nil || scope.conflict || scope.disallowLearning { + if scope.userID <= 0 || scope.keyID <= 0 || scope.userID != h.anchor.user || scope.keyID != h.anchor.key || scope.groupID != h.anchor.group || scope.session != h.anchor.session || scope.lookupErr != nil || scope.conflict || scope.disallowLearning { return err } delta, ok := openAIWSPlainItems(source) From 976420004f8b016647799850e5fccb7f5981abed Mon Sep 17 00:00:00 2001 From: ranxi2001 Date: Mon, 16 Nov 2026 17:33:06 +0800 Subject: [PATCH 4/9] =?UTF-8?q?fix:=20=E5=8F=AF=E8=AF=BB=E7=BB=AD=E6=8E=A5?= =?UTF-8?q?=E4=BF=9D=E7=95=99=E5=8E=86=E5=8F=B2=E6=8E=A8=E7=90=86=E6=91=98?= =?UTF-8?q?=E8=A6=81=E5=B9=B6=E6=8B=92=E7=BB=9D=E5=B5=8C=E5=A5=97=E5=AF=86?= =?UTF-8?q?=E6=96=87?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../handler/openai_ws_unsent_delta_test.go | 4 +- .../service/openai_ws_agent_recovery.go | 54 ++++++++++++++++--- 2 files changed, 49 insertions(+), 9 deletions(-) diff --git a/backend/internal/handler/openai_ws_unsent_delta_test.go b/backend/internal/handler/openai_ws_unsent_delta_test.go index 2a35cc4d83e9..0d6418749d39 100644 --- a/backend/internal/handler/openai_ws_unsent_delta_test.go +++ b/backend/internal/handler/openai_ws_unsent_delta_test.go @@ -116,7 +116,7 @@ func runUnsentDelta(t *testing.T, mode string, switchTurn int, fence string) { raw, _ := json.Marshal(b) return string(raw) } - issue715WSSend(t, h.clientConn, create(`[{"role":"user","content":"Perform fixture exactly once and retain the task"}]`, "")) + issue715WSSend(t, h.clientConn, create(`[{"role":"user","content":"Perform fixture exactly once and retain the task"},{"type":"reasoning","encrypted_content":"fixture-history-state","summary":[{"type":"summary_text","text":"fixture readable progress"}]}]`, "")) executions := 0 consume := func(raw []byte) { for _, item := range gjson.GetBytes(raw, "response.output").Array() { @@ -172,6 +172,8 @@ func runUnsentDelta(t *testing.T, mode string, switchTurn int, fence string) { require.NotContains(t, string(resumed), "resp_delta_", "old provider pointer must not reach B") require.Contains(t, string(resumed), "Perform fixture exactly once") require.Contains(t, string(resumed), "fixture applied exactly once") + require.Contains(t, string(resumed), "fixture readable progress") + require.NotContains(t, string(resumed), "fixture-history-state") require.Contains(t, string(resumed), "9007199254740993") require.True(t, gjson.GetBytes(resumed, "tools").IsArray(), "effective tool declarations must survive takeover") for _, item := range gjson.GetBytes(resumed, "input").Array() { diff --git a/backend/internal/service/openai_ws_agent_recovery.go b/backend/internal/service/openai_ws_agent_recovery.go index a0d4577cb37b..3ef673edc380 100644 --- a/backend/internal/service/openai_ws_agent_recovery.go +++ b/backend/internal/service/openai_ws_agent_recovery.go @@ -649,6 +649,21 @@ func openAIWSPlainItems(body []byte) ([]json.RawMessage, bool) { if !utf8.Valid(body) || !openAIEncryptedJSONUnambiguous(body) { return nil, false } + // Historical reasoning can contain a readable summary plus provider state. + // Apply the existing explicit Beta representation, preserving supplied text + // and marking unavailable state. Tool data and current agent envelopes are + // still rejected below; this never fabricates a task from ciphertext. + hasReasoning := false + for _, item := range gjson.GetBytes(body, "input").Array() { + hasReasoning = hasReasoning || item.Get("type").String() == "reasoning" + } + if hasReasoning { + var e error + body, e = prepareOpenAI409ReadableReplay(body) + if e != nil || len(body) > openAIWSLocalHistoryLimit { + return nil, false + } + } input := gjson.GetBytes(body, "input") var items []json.RawMessage if input.Type == gjson.String { @@ -672,7 +687,7 @@ func openAIWSPlainItems(body []byte) ([]json.RawMessage, bool) { } func openAIWSPlainItem(raw []byte) bool { item := gjson.ParseBytes(raw) - if !item.IsObject() || item.Get("encrypted_content").Exists() { + if !item.IsObject() || openAIWSHasOpaqueField(item, 0) { return false } switch item.Get("type").String() { @@ -764,15 +779,23 @@ func (r *openAIWSAgentRecovery) commitLocalHistory(completed bool) { if h.output.finalItems != nil { output = h.output.finalItems } - for _, item := range output { - if !openAIWSPlainItem(item) { - h.ready = false - h.committed = nil - return - } + if output == nil { + output = []json.RawMessage{} + } + encoded, e := json.Marshal(output) + if e != nil { + h.ready = false + h.committed = nil + return } + normalized, ok := openAIWSPlainItems(append(append([]byte(`{"input":`), encoded...), '}')) + if !ok { + h.ready = false + h.committed = nil + return + } + output = normalized all := combineOpenAIWSReplayItems(h.pending, output) - var ok bool h.committed, ok = openAIWSHistoryCopy(all) h.ready = ok if ok { @@ -900,3 +923,18 @@ func (r *openAIWSAgentRecovery) accountLocalHistory() { } r.historyReserved = wanted } + +func openAIWSHasOpaqueField(value gjson.Result, depth int) bool { + if depth > 64 { + return true + } + if !value.IsObject() && !value.IsArray() { + return false + } + opaque := false + value.ForEach(func(key, child gjson.Result) bool { + opaque = key.String() == "encrypted_content" || openAIWSHasOpaqueField(child, depth+1) + return !opaque + }) + return opaque +} From 6394893e14ef0a4eafbe343ba5916a2befc17dfa Mon Sep 17 00:00:00 2001 From: ranxi2001 Date: Mon, 16 Nov 2026 17:34:04 +0800 Subject: [PATCH 5/9] =?UTF-8?q?test:=20=E9=95=BF=E8=BD=AE=E6=AC=A1?= =?UTF-8?q?=E5=A4=B9=E5=85=B7=E9=80=90=E8=BD=AE=E6=B6=88=E8=B4=B9=E7=94=A8?= =?UTF-8?q?=E9=87=8F=E4=BB=A5=E9=81=BF=E5=85=8D=E6=B5=8B=E8=AF=95=E9=98=9F?= =?UTF-8?q?=E5=88=97=E9=98=BB=E5=A1=9E?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../internal/handler/openai_ws_unsent_delta_test.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/backend/internal/handler/openai_ws_unsent_delta_test.go b/backend/internal/handler/openai_ws_unsent_delta_test.go index 0d6418749d39..1e68885da2d7 100644 --- a/backend/internal/handler/openai_ws_unsent_delta_test.go +++ b/backend/internal/handler/openai_ws_unsent_delta_test.go @@ -106,6 +106,13 @@ func runUnsentDelta(t *testing.T, mode string, switchTurn int, fence string) { *repo = accounts }) defer issue600Finish(t, h, headers) + awaitUsage := func() { + select { + case <-h.usageLogs: + case <-time.After(5 * time.Second): + t.Fatal("completed fixture turn did not settle usage") + } + } create := func(input, previous string) string { b := map[string]any{"type": "response.create", "model": "gpt-5.1", "store": true, "input": json.RawMessage(input)} if previous != "" { @@ -128,11 +135,13 @@ func runUnsentDelta(t *testing.T, mode string, switchTurn int, fence string) { } first := issue715WSRead(t, h.clientConn) consume(first) + awaitUsage() require.Equal(t, "response.completed", gjson.GetBytes(first, "type").String()) for n := 2; n < switchTurn; n++ { issue715WSSend(t, h.clientConn, create(fmt.Sprintf(`[{"role":"user","content":"fixture next %d"}]`, n), fmt.Sprintf("resp_delta_%d", n-1))) round := issue715WSRead(t, h.clientConn) consume(round) + awaitUsage() require.Equal(t, "response.completed", gjson.GetBytes(round, "type").String()) } accounts.disable(1) @@ -162,6 +171,7 @@ func runUnsentDelta(t *testing.T, mode string, switchTurn int, fence string) { logReadableWSRecoveryDetails(t, h) } require.Equal(t, "response.completed", gjson.GetBytes(result, "type").String(), "UNSENT_DELTA_NOT_RESUMED: %s", result) + awaitUsage() mu.Lock() sent := append([][]byte(nil), frames...) selected := append([]string(nil), owners...) @@ -187,6 +197,7 @@ func runUnsentDelta(t *testing.T, mode string, switchTurn int, fence string) { issue715WSSend(t, h.clientConn, create(`[{"role":"user","content":"fixture final followup"}]`, fmt.Sprintf("resp_delta_%d", switchTurn))) round := issue715WSRead(t, h.clientConn) consume(round) + awaitUsage() require.Equal(t, "response.completed", gjson.GetBytes(round, "type").String()) mu.Lock() defer mu.Unlock() From 5d7f2f896daf20c4ec53c57606e364f1cf505b6d Mon Sep 17 00:00:00 2001 From: ranxi2001 Date: Mon, 16 Nov 2026 17:34:54 +0800 Subject: [PATCH 6/9] =?UTF-8?q?fix:=20=E8=BF=9E=E6=8E=A5=E5=8E=86=E5=8F=B2?= =?UTF-8?q?=E7=8B=AC=E7=AB=8B=E9=81=B5=E5=AE=88=E5=AE=8C=E6=95=B4=E7=BB=88?= =?UTF-8?q?=E6=80=81=E8=BE=93=E5=87=BA=E7=9A=84=E4=BC=98=E5=85=88=E7=BA=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../service/openai_ws_agent_recovery.go | 51 ++++++++++++++++--- 1 file changed, 45 insertions(+), 6 deletions(-) diff --git a/backend/internal/service/openai_ws_agent_recovery.go b/backend/internal/service/openai_ws_agent_recovery.go index 3ef673edc380..231039610abb 100644 --- a/backend/internal/service/openai_ws_agent_recovery.go +++ b/backend/internal/service/openai_ws_agent_recovery.go @@ -629,7 +629,7 @@ type openAIWSLocalHistory struct { instructions json.RawMessage committed []json.RawMessage pending []json.RawMessage - output openAIWSToolCallReplayCollector + output openAIWSLocalOutput anchor openAIWSBridgeResponseAnchor clientModel string ready bool @@ -723,7 +723,7 @@ func (r *openAIWSAgentRecovery) stageLocalHistory(source []byte) { defer r.accountLocalHistory() r.localHistory.pending = nil r.localHistory.outputBytes = 0 - r.localHistory.output = openAIWSToolCallReplayCollector{} + r.localHistory.output = openAIWSLocalOutput{} if r.service.settingService == nil || !r.service.settingService.IsOpenAI409OmitEncryptedHistoryEnabled(r.ctx) { r.localHistory = openAIWSLocalHistory{} return @@ -775,9 +775,9 @@ func (r *openAIWSAgentRecovery) commitLocalHistory(completed bool) { h.committed = nil return } - output := h.output.AllItems() - if h.output.finalItems != nil { - output = h.output.finalItems + output := h.output.items + if h.output.final != nil { + output = h.output.final } if output == nil { output = []json.RawMessage{} @@ -803,7 +803,7 @@ func (r *openAIWSAgentRecovery) commitLocalHistory(completed bool) { } h.pending = nil h.outputBytes = 0 - h.output = openAIWSToolCallReplayCollector{} + h.output = openAIWSLocalOutput{} } // Called at the outer proxy return boundary, after transport goroutines stop. @@ -938,3 +938,42 @@ func openAIWSHasOpaqueField(value gjson.Result, depth int) bool { }) return opaque } + +// Kept with the manager: protocol regression baselines may restore an older +// bridge collector, while this socket history still needs terminal authority. +type openAIWSLocalOutput struct { + items, final []json.RawMessage + seen map[string]bool +} + +func (o *openAIWSLocalOutput) AddEvent(event string, payload []byte) { + var items []gjson.Result + if event == "response.output_item.done" { + items = []gjson.Result{gjson.GetBytes(payload, "item")} + } + if (event == "response.completed" || event == "response.done") && openAIAgentCompletedResponse(payload, true) { + items = gjson.GetBytes(payload, "response.output").Array() + if len(items) > 0 { + o.final = []json.RawMessage{} + for _, v := range items { + o.final = append(o.final, append(json.RawMessage(nil), v.Raw...)) + } + } + } + for _, v := range items { + key := v.Get("id").String() + if key == "" { + key = v.Get("call_id").String() + } + if key == "" { + key = v.Raw + } + if o.seen == nil { + o.seen = map[string]bool{} + } + if !o.seen[key] { + o.seen[key] = true + o.items = append(o.items, append(json.RawMessage(nil), v.Raw...)) + } + } +} From 4b13c785786ff1d623bc23203991dac405494567 Mon Sep 17 00:00:00 2001 From: ranxi2001 Date: Mon, 16 Nov 2026 17:39:12 +0800 Subject: [PATCH 7/9] =?UTF-8?q?fix:=20=E8=A1=A5=E6=A1=A5=E6=8E=A5=E5=B0=BE?= =?UTF-8?q?=E9=83=A8=E7=BB=93=E6=9E=84=E8=AF=8A=E6=96=AD=E5=B9=B6=E6=A0=B8?= =?UTF-8?q?=E5=AF=B9=E6=8B=92=E7=BB=9D=E5=90=8E=E4=B8=8D=E5=86=8D=E7=94=9F?= =?UTF-8?q?=E6=88=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../handler/openai_ws_unsent_delta_test.go | 13 +++++++++++++ .../service/openai_ws_agent_recovery.go | 19 +++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/backend/internal/handler/openai_ws_unsent_delta_test.go b/backend/internal/handler/openai_ws_unsent_delta_test.go index 1e68885da2d7..a06b2ee6e849 100644 --- a/backend/internal/handler/openai_ws_unsent_delta_test.go +++ b/backend/internal/handler/openai_ws_unsent_delta_test.go @@ -105,6 +105,10 @@ func runUnsentDelta(t *testing.T, mode string, switchTurn int, fence string) { accounts = &readableWSAccounts{issue600SwitchRepo: &issue600SwitchRepo{&openAIWSFailoverHandlerAccountRepoStub{accounts: []service.Account{*a, b}}}} *repo = accounts }) + completion := h.handlerDone + finished := make(chan struct{}) + h.handlerDone = finished + go func() { <-completion; close(finished) }() defer issue600Finish(t, h, headers) awaitUsage := func() { select { @@ -158,6 +162,15 @@ func runUnsentDelta(t *testing.T, mode string, switchTurn int, fence string) { readCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() _, raw, e := h.clientConn.Read(readCtx) + if e == nil { + require.Equal(t, "error", gjson.GetBytes(raw, "type").String()) + _, _, e = h.clientConn.Read(readCtx) + } + select { + case <-finished: + case <-readCtx.Done(): + t.Fatal("refusal did not finish; no post-check generation permitted") + } mu.Lock() n := len(frames) mu.Unlock() diff --git a/backend/internal/service/openai_ws_agent_recovery.go b/backend/internal/service/openai_ws_agent_recovery.go index 231039610abb..d08a318b912f 100644 --- a/backend/internal/service/openai_ws_agent_recovery.go +++ b/backend/internal/service/openai_ws_agent_recovery.go @@ -606,7 +606,26 @@ func openAIWSWireRefusal(ctx context.Context, stage string, source, payload []by storeKind = "other" } } + sanitized, idsChanged, sanitizeErr := sanitizeOpenAIResponsesInputItemIDs(source) + tailMatchesSource, tailMatchesIDs := false, false + if sourceInput.IsArray() && wireInput.IsArray() { + original, wire := sourceInput.Array(), wireInput.Array() + if len(original) > 0 && len(wire) >= len(original) { + var tail []json.RawMessage + for _, item := range wire[len(wire)-len(original):] { + tail = append(tail, json.RawMessage(item.Raw)) + } + encoded, e := json.Marshal(tail) + if e == nil { + tailBody := append(append([]byte(`{"input":`), encoded...), '}') + tailMatchesSource = openAIAgentRecoveryInputDigest(source) == openAIAgentRecoveryInputDigest(tailBody) + tailMatchesIDs = sanitizeErr == nil && openAIAgentRecoveryInputDigest(sanitized) == openAIAgentRecoveryInputDigest(tailBody) + } + } + } logger.FromContext(ctx).Warn("openai.encrypted_history_409_refused", zap.String("stage", stage), + zap.Bool("source_ids_normalized", idsChanged), zap.Bool("tail_matches_source", tailMatchesSource), zap.Bool("tail_matches_normalized_ids", tailMatchesIDs), + zap.Int("source_agent_items", len(openAIAgentRecoveryItemDigests(source))), zap.Int("wire_agent_items", len(openAIAgentRecoveryItemDigests(payload))), zap.String("source_input_kind", inputKind), zap.String("source_store", storeKind), zap.Int("source_input_items", len(sourceInput.Array())), zap.Int("wire_input_items", len(wireInput.Array())), zap.Bool("source_previous_present", gjson.GetBytes(source, "previous_response_id").String() != ""), From eb86f48c15fda24c336a28ab4a916f988300e0c1 Mon Sep 17 00:00:00 2001 From: ranxi2001 Date: Mon, 16 Nov 2026 18:00:05 +0800 Subject: [PATCH 8/9] fix: preserve WS current-frame instructions and continuation anchors --- .github/workflows/ws-unsent-delta.yml | 81 ++++++++++++ .../handler/openai_ws_unsent_delta_test.go | 56 +++++++-- .../service/openai_ws_agent_recovery.go | 49 ++++---- .../service/openai_ws_unsent_delta_test.go | 117 +++++++++++++++++- 4 files changed, 265 insertions(+), 38 deletions(-) diff --git a/.github/workflows/ws-unsent-delta.yml b/.github/workflows/ws-unsent-delta.yml index 6bd80e350cb1..ba13567ee539 100644 --- a/.github/workflows/ws-unsent-delta.yml +++ b/.github/workflows/ws-unsent-delta.yml @@ -82,3 +82,84 @@ jobs: name: unsent-delta-${{ matrix.phase }}-${{ github.event.pull_request.head.sha }} path: ${{ runner.temp }}/unsent-delta*.jsonl if-no-files-found: error + repair877: + runs-on: ubuntu-24.04 + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + phase: [RED, GREEN] + steps: + - uses: actions/checkout@v6 + with: + ref: ${{ github.event.pull_request.head.sha }} + fetch-depth: 0 + persist-credentials: false + - uses: actions/setup-go@v6 + with: + go-version-file: backend/go.mod + cache-dependency-path: backend/go.sum + - name: Exact reviewed PR877 implementation for repair RED + if: matrix.phase == 'RED' + shell: bash + run: | + set -euo pipefail + # Keep all current tests. Replace only the implementation containing + # the two reviewed faults; the original takeover RED stays separate. + baseline=4b13c785786ff1d623bc23203991dac405494567 + target=backend/internal/service/openai_ws_agent_recovery.go + git show "$baseline:$target" > "$target" + test "$(git hash-object "$target")" = "$(git rev-parse "$baseline:$target")" + - name: Named instruction and continuation-anchor repair evidence + env: + PHASE: ${{ matrix.phase }} + shell: bash + run: | + set -euo pipefail + set +e + (cd backend && go test -race -json -tags=unit,unsentdelta ./internal/service -run '^TestOpenAIWSUnsentDelta(Instructions|ContinuationAnchor|DeliveryScope)$' -count=1 -timeout=3m) > "$RUNNER_TEMP/unsent-delta-repair877.jsonl" 2>&1 + result=$? + set -e + cat "$RUNNER_TEMP/unsent-delta-repair877.jsonl" + python3 .github/scripts/assert-readable-continuation.py --clean-log "$RUNNER_TEMP/unsent-delta-repair877.jsonl" + python3 - "$PHASE" "$result" "$RUNNER_TEMP/unsent-delta-repair877.jsonl" <<'PY' + import json,sys + from collections import Counter + phase,status,path=sys.argv[1],int(sys.argv[2]),sys.argv[3] + rows=[] + for line in open(path): + try: rows.append(json.loads(line)) + except ValueError: pass + package='github.com/Wei-Shaw/sub2api/internal/service' + instructions='TestOpenAIWSUnsentDeltaInstructions' + anchor='TestOpenAIWSUnsentDeltaContinuationAnchor' + delivery='TestOpenAIWSUnsentDeltaDeliveryScope' + roots={instructions,anchor,delivery} + leaves={instructions+'/'+s for s in ('omitted_after_omitted_turn','omitted_after_explicit_turn','current_null','current_explicit','current_empty')} + leaves.update(anchor+'/'+s for s in ('missing_anchor_independent_task','empty_anchor_independent_task','null_anchor_independent_task','valid_anchor_independent_task','valid_anchor_empty_previous','valid_anchor_null_previous','valid_anchor_blank_previous','valid_anchor_wrong_previous','completed_matching_anchor')) + leaves.update(delivery+'/'+s for s in ('valid','pending','failed_write','terminal_excludes','unknown_call','changed_call','wrong_pointer','wrong_key','wrong_model','sent_error','oversize')) + markers={instructions+'/'+s:'STALE_INSTRUCTIONS_RESTORED' for s in ('omitted_after_omitted_turn','omitted_after_explicit_turn')} + markers.update({anchor+'/'+s:'INDEPENDENT_TASK_ATTACHED' for s in ('missing_anchor_independent_task','empty_anchor_independent_task','null_anchor_independent_task')}) + assert phase in ('RED','GREEN'),phase + failures=set(markers)|{instructions,anchor,None} if phase=='RED' else set() + counts=Counter((r.get('Action'),r.get('Test')) for r in rows if r.get('Package')==package) + assert status==(1 if phase=='RED' else 0),status + assert all(r.get('Package')==package for r in rows),rows + assert {r.get('Test') for r in rows if r.get('Action')=='run'}==leaves|roots,rows + assert {r.get('Test') for r in rows if r.get('Action')=='fail'}==failures,rows + assert not any(r.get('Action')=='skip' for r in rows),rows + for name in leaves|roots|{None}: + assert name is None or counts['run',name]==1,(name,counts) + expected='fail' if name in failures else 'pass' + assert counts[expected,name]==1 and counts['pass',name]+counts['fail',name]==1,(name,counts) + if phase=='RED': + for name,marker in markers.items(): + output=''.join(r.get('Output','') for r in rows if r.get('Test')==name) + assert marker in output,(name,output) + PY + - uses: actions/upload-artifact@v7 + if: always() + with: + name: unsent-delta-repair877-${{ matrix.phase }}-${{ github.event.pull_request.head.sha }} + path: ${{ runner.temp }}/unsent-delta-repair877.jsonl + if-no-files-found: error diff --git a/backend/internal/handler/openai_ws_unsent_delta_test.go b/backend/internal/handler/openai_ws_unsent_delta_test.go index a06b2ee6e849..c6d94adc8472 100644 --- a/backend/internal/handler/openai_ws_unsent_delta_test.go +++ b/backend/internal/handler/openai_ws_unsent_delta_test.go @@ -40,7 +40,7 @@ func runUnsentDelta(t *testing.T, mode string, switchTurn int, fence string) { var mu sync.Mutex var frames [][]byte var owners []string - response := func(body []byte, auth string) []byte { + response := func(body []byte, auth string) [][]byte { mu.Lock() defer mu.Unlock() frames = append(frames, append([]byte(nil), body...)) @@ -50,10 +50,14 @@ func runUnsentDelta(t *testing.T, mode string, switchTurn int, fence string) { if n == switchTurn-1 { output = `[{"type":"function_call","call_id":"call_takeover_once","name":"fixture","arguments":"{\"n\":9007199254740993}","status":"completed"}]` } + var events [][]byte if n == switchTurn-1 && fence == "terminal_excludes" { - output = `[{"type":"message","role":"assistant","content":[{"type":"output_text","text":"no tool delivered"}]}]` + // Deliver the real call first. The nonempty terminal is authoritative + // and excludes that call even though the client already received it. + events = append(events, []byte(`{"type":"response.output_item.done","item":`+gjson.Get(output, "0").Raw+`}`)) + output = `[{"type":"message","role":"assistant","content":[{"type":"output_text","text":"fixture terminal excludes delivered tool"}]}]` } - return []byte(fmt.Sprintf(`{"type":"response.completed","response":{"id":"resp_delta_%d","model":"gpt-5.1","status":"completed","output":%s,"usage":{"input_tokens":5,"output_tokens":3}}}`, n, output)) + return append(events, []byte(fmt.Sprintf(`{"type":"response.completed","response":{"id":"resp_delta_%d","model":"gpt-5.1","status":"completed","output":%s,"usage":{"input_tokens":5,"output_tokens":3}}}`, n, output))) } upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if !strings.EqualFold(r.Header.Get("Upgrade"), "websocket") { @@ -63,7 +67,9 @@ func runUnsentDelta(t *testing.T, mode string, switchTurn int, fence string) { return } w.Header().Set("Content-Type", "text/event-stream") - fmt.Fprintf(w, "data: %s\n\n", response(body, r.Header.Get("Authorization"))) + for _, event := range response(body, r.Header.Get("Authorization")) { + fmt.Fprintf(w, "data: %s\n\n", event) + } return } conn, e := coderws.Accept(w, r, nil) @@ -77,8 +83,10 @@ func runUnsentDelta(t *testing.T, mode string, switchTurn int, fence string) { if e != nil { return } - if conn.Write(r.Context(), coderws.MessageText, response(body, r.Header.Get("Authorization"))) != nil { - return + for _, event := range response(body, r.Header.Get("Authorization")) { + if conn.Write(r.Context(), coderws.MessageText, event) != nil { + return + } } } })) @@ -130,20 +138,50 @@ func runUnsentDelta(t *testing.T, mode string, switchTurn int, fence string) { issue715WSSend(t, h.clientConn, create(`[{"role":"user","content":"Perform fixture exactly once and retain the task"},{"type":"reasoning","encrypted_content":"fixture-history-state","summary":[{"type":"summary_text","text":"fixture readable progress"}]}]`, "")) executions := 0 consume := func(raw []byte) { - for _, item := range gjson.GetBytes(raw, "response.output").Array() { + items := gjson.GetBytes(raw, "response.output").Array() + if gjson.GetBytes(raw, "type").String() == "response.output_item.done" { + items = append(items, gjson.GetBytes(raw, "item")) + } + for _, item := range items { if item.Get("type").String() == "function_call" { require.Equal(t, "call_takeover_once", item.Get("call_id").String()) executions++ } } } - first := issue715WSRead(t, h.clientConn) + excludedToolDeliveries := 0 + readTurn := func() []byte { + if fence != "terminal_excludes" { + return issue715WSRead(t, h.clientConn) + } + ctx, cancel := context.WithTimeout(context.Background(), 8*time.Second) + defer cancel() + for { + _, event, err := h.clientConn.Read(ctx) + require.NoError(t, err) + kind := gjson.GetBytes(event, "type").String() + if kind == "response.output_item.done" && gjson.GetBytes(event, "item.type").String() == "function_call" { + require.Equal(t, "call_takeover_once", gjson.GetBytes(event, "item.call_id").String()) + excludedToolDeliveries++ + consume(event) + } + if kind == "response.completed" || kind == "response.failed" || kind == "error" { + return event + } + } + } + first := readTurn() consume(first) awaitUsage() require.Equal(t, "response.completed", gjson.GetBytes(first, "type").String()) + if fence == "terminal_excludes" { + require.Equal(t, 1, excludedToolDeliveries, "terminal exclusion must follow actual tool item delivery to the client") + require.Equal(t, 1, executions) + require.NotContains(t, string(first), "call_takeover_once", "the nonempty terminal must exclude the delivered call") + } for n := 2; n < switchTurn; n++ { issue715WSSend(t, h.clientConn, create(fmt.Sprintf(`[{"role":"user","content":"fixture next %d"}]`, n), fmt.Sprintf("resp_delta_%d", n-1))) - round := issue715WSRead(t, h.clientConn) + round := readTurn() consume(round) awaitUsage() require.Equal(t, "response.completed", gjson.GetBytes(round, "type").String()) diff --git a/backend/internal/service/openai_ws_agent_recovery.go b/backend/internal/service/openai_ws_agent_recovery.go index d08a318b912f..fd7d062e250c 100644 --- a/backend/internal/service/openai_ws_agent_recovery.go +++ b/backend/internal/service/openai_ws_agent_recovery.go @@ -8,6 +8,7 @@ import ( "io" "net/http" "net/url" + "strings" "sync" "sync/atomic" "time" @@ -643,15 +644,14 @@ const openAIWSLocalHistoryLimit = 8 << 20 var openAIWSLocalHistoryBytes atomic.Int64 type openAIWSLocalHistory struct { - outputBytes int - tools json.RawMessage - instructions json.RawMessage - committed []json.RawMessage - pending []json.RawMessage - output openAIWSLocalOutput - anchor openAIWSBridgeResponseAnchor - clientModel string - ready bool + outputBytes int + tools json.RawMessage + committed []json.RawMessage + pending []json.RawMessage + output openAIWSLocalOutput + anchor openAIWSBridgeResponseAnchor + clientModel string + ready bool } type openAIWSRestoredUnsentError struct{ *openAIWSUnsentTurnError } @@ -766,14 +766,13 @@ func (r *openAIWSAgentRecovery) stageLocalHistory(source []byte) { } if previous == "" { r.localHistory.tools = nil - r.localHistory.instructions = nil } - for field, dst := range map[string]*json.RawMessage{"tools": &r.localHistory.tools, "instructions": &r.localHistory.instructions} { - if value := gjson.GetBytes(source, field); value.Exists() { - *dst = append(json.RawMessage(nil), value.Raw...) - } + // Tools retain the socket's declarations. Instructions apply only to the + // current frame, including when omitted, and must never enter this history. + if value := gjson.GetBytes(source, "tools"); value.Exists() { + r.localHistory.tools = append(json.RawMessage(nil), value.Raw...) } - if len(r.localHistory.tools)+len(r.localHistory.instructions) > openAIWSLocalHistoryLimit { + if len(r.localHistory.tools) > openAIWSLocalHistoryLimit { r.localHistory = openAIWSLocalHistory{} return } @@ -789,7 +788,7 @@ func (r *openAIWSAgentRecovery) stageLocalHistory(source []byte) { func (r *openAIWSAgentRecovery) commitLocalHistory(completed bool) { defer r.accountLocalHistory() h := &r.localHistory - if !completed || h.pending == nil { + if !completed || h.pending == nil || strings.TrimSpace(r.completedBridge.response) == "" { h.ready = false h.committed = nil return @@ -844,7 +843,11 @@ func (r *openAIWSAgentRecovery) restoreUnsentHistory(err error) error { return err } source := unsent.body - if len(source) > openAIWSLocalHistoryLimit || gjson.GetBytes(source, "previous_response_id").String() != h.anchor.response || gjson.GetBytes(source, "conversation").Exists() { + previous := gjson.GetBytes(source, "previous_response_id") + // Two blank IDs describe no continuation. Only the client's explicit + // pointer to a delivered, completed response may attach another turn. + if len(source) > openAIWSLocalHistoryLimit || previous.Type != gjson.String || strings.TrimSpace(previous.String()) == "" || + strings.TrimSpace(h.anchor.response) == "" || previous.String() != h.anchor.response || gjson.GetBytes(source, "conversation").Exists() { return err } if unsent.model != "" && unsent.model != h.clientModel { @@ -886,12 +889,10 @@ func (r *openAIWSAgentRecovery) restoreUnsentHistory(err error) error { return err } } - for field, value := range map[string]json.RawMessage{"tools": h.tools, "instructions": h.instructions} { - if !gjson.GetBytes(expanded, field).Exists() && len(value) > 0 { - expanded, e = sjson.SetRawBytes(expanded, field, value) - if e != nil { - return err - } + if !gjson.GetBytes(expanded, "tools").Exists() && len(h.tools) > 0 { + expanded, e = sjson.SetRawBytes(expanded, "tools", h.tools) + if e != nil { + return err } } if !openAI409HasReadableIntent(expanded) { @@ -917,7 +918,7 @@ func (r *openAIWSAgentRecovery) restoreUnsentHistory(err error) error { // optional restoration path. It never rejects normal forwarding or drops bytes. func (r *openAIWSAgentRecovery) accountLocalHistory() { h := &r.localHistory - wanted := int64(len(h.tools) + len(h.instructions) + 3*h.outputBytes) + wanted := int64(len(h.tools) + 3*h.outputBytes) for _, set := range [][]json.RawMessage{h.pending, h.committed} { for _, v := range set { wanted += int64(len(v)) diff --git a/backend/internal/service/openai_ws_unsent_delta_test.go b/backend/internal/service/openai_ws_unsent_delta_test.go index 429146d75e20..3ca371c2655b 100644 --- a/backend/internal/service/openai_ws_unsent_delta_test.go +++ b/backend/internal/service/openai_ws_unsent_delta_test.go @@ -5,10 +5,12 @@ package service import ( "context" "errors" - "github.com/Wei-Shaw/sub2api/internal/config" - "github.com/stretchr/testify/require" "strings" "testing" + + "github.com/Wei-Shaw/sub2api/internal/config" + "github.com/stretchr/testify/require" + "github.com/tidwall/gjson" ) func TestOpenAIWSUnsentDeltaDeliveryScope(t *testing.T) { @@ -25,9 +27,15 @@ func TestOpenAIWSUnsentDeltaDeliveryScope(t *testing.T) { source := `{"model":"gpt-6-astra","input":[{"role":"user","content":"fixture original task"}]}` require.NoError(t, issue715WSBegin(t, manager, a, source, "ctx_pool")) call := `{"type":"function_call","call_id":"call_fixture_delta","name":"fixture","arguments":"{}","status":"completed"}` - if mutation == "pending" { - require.NoError(t, manager.write([]byte(`{"type":"response.output_item.done","item":`+call+`}`), func() error { return nil })) - } else { + if mutation == "pending" || mutation == "terminal_excludes" { + delivered := false + require.NoError(t, manager.write([]byte(`{"type":"response.output_item.done","item":`+call+`}`), func() error { + delivered = true + return nil + })) + require.True(t, delivered, "the tool item must reach the client before its terminal verdict") + } + if mutation != "pending" { output := call if mutation == "terminal_excludes" { output = `{"type":"message","role":"assistant","content":"fixture complete without tool"}` @@ -79,3 +87,102 @@ func TestOpenAIWSUnsentDeltaDeliveryScope(t *testing.T) { }) } } + +func TestOpenAIWSUnsentDeltaInstructions(t *testing.T) { + for _, tc := range []struct { + name string + second string + third string + }{ + {"omitted_after_omitted_turn", "", ""}, + {"omitted_after_explicit_turn", `,"instructions":"fixture turn two only"`, ""}, + {"current_null", "", `,"instructions":null`}, + {"current_explicit", "", `,"instructions":"fixture current control"`}, + {"current_empty", "", `,"instructions":""`}, + } { + t.Run(tc.name, func(t *testing.T) { + before := openAIWSLocalHistoryBytes.Load() + svc := &OpenAIGatewayService{cfg: &config.Config{}, settingService: NewSettingService(&codexPolicyMigrationRepoStub{values: map[string]string{SettingKeyOpenAI409OmitEncryptedHistoryEnabled: "true"}}, nil)} + manager, c, account := issue715WSManager(t, svc, context.Background()) + SetOpenAIWSRecoveryPrincipal(c, 715, 715) + defer func() { + manager.abort() + require.Equal(t, before, openAIWSLocalHistoryBytes.Load()) + }() + first := `{"model":"gpt-6-astra","instructions":"fixture turn one only","tools":[{"type":"function","name":"fixture","parameters":{"type":"object"}}],"input":"fixture original task"}` + require.NoError(t, issue715WSBegin(t, manager, account, first, "ctx_pool")) + require.NoError(t, manager.write([]byte(`{"type":"response.completed","response":{"id":"resp_instructions_1","status":"completed","output":[{"type":"message","role":"assistant","content":"fixture first answer"}]}}`), func() error { return nil })) + second := `{"model":"gpt-6-astra","previous_response_id":"resp_instructions_1","input":"fixture second task"` + tc.second + `}` + require.NoError(t, issue715WSBegin(t, manager, account, second, "ctx_pool")) + require.NoError(t, manager.write([]byte(`{"type":"response.completed","response":{"id":"resp_instructions_2","status":"completed","output":[{"type":"message","role":"assistant","content":"fixture second answer"}]}}`), func() error { return nil })) + third := `{"type":"response.create","model":"gpt-6-astra","previous_response_id":"resp_instructions_2","input":"fixture current task"` + tc.third + `}` + cause := openAIWSUnsentTurn(denyOpenAITurn("account_ineligible"), []byte(third), "gpt-6-astra", 3) + var restored *openAIWSRestoredUnsentError + require.ErrorAs(t, manager.restoreUnsentHistory(cause), &restored) + want, got := gjson.Get(third, "instructions"), gjson.GetBytes(restored.body, "instructions") + require.Equal(t, want.Exists(), got.Exists(), "STALE_INSTRUCTIONS_RESTORED: preserve current-frame field presence") + require.Equal(t, want.Raw, got.Raw, "STALE_INSTRUCTIONS_RESTORED: preserve current-frame null or explicit value") + require.Equal(t, gjson.Get(first, "tools").Raw, gjson.GetBytes(restored.body, "tools").Raw, "socket tool declarations still survive takeover") + for _, text := range []string{"fixture original task", "fixture first answer", "fixture second task", "fixture second answer", "fixture current task"} { + require.Contains(t, string(restored.body), text) + } + require.NotContains(t, string(restored.body), "previous_response_id") + require.Same(t, cause, manager.restoreUnsentHistory(cause), "restoration proof is consumed once") + }) + } +} + +func TestOpenAIWSUnsentDeltaContinuationAnchor(t *testing.T) { + for _, tc := range []struct { + name string + response string + previous string + want bool + }{ + {"missing_anchor_independent_task", "", "", false}, + {"empty_anchor_independent_task", `"id":"",`, `,"previous_response_id":""`, false}, + {"null_anchor_independent_task", `"id":null,`, `,"previous_response_id":null`, false}, + {"valid_anchor_independent_task", `"id":"resp_anchor",`, "", false}, + {"valid_anchor_empty_previous", `"id":"resp_anchor",`, `,"previous_response_id":""`, false}, + {"valid_anchor_null_previous", `"id":"resp_anchor",`, `,"previous_response_id":null`, false}, + {"valid_anchor_blank_previous", `"id":"resp_anchor",`, `,"previous_response_id":" "`, false}, + {"valid_anchor_wrong_previous", `"id":"resp_anchor",`, `,"previous_response_id":"resp_foreign"`, false}, + {"completed_matching_anchor", `"id":"resp_anchor",`, `,"previous_response_id":"resp_anchor"`, true}, + } { + t.Run(tc.name, func(t *testing.T) { + before := openAIWSLocalHistoryBytes.Load() + svc := &OpenAIGatewayService{cfg: &config.Config{}, settingService: NewSettingService(&codexPolicyMigrationRepoStub{values: map[string]string{SettingKeyOpenAI409OmitEncryptedHistoryEnabled: "true"}}, nil)} + manager, c, account := issue715WSManager(t, svc, context.Background()) + SetOpenAIWSRecoveryPrincipal(c, 715, 715) + defer func() { + manager.abort() + require.Equal(t, before, openAIWSLocalHistoryBytes.Load()) + }() + require.NoError(t, issue715WSBegin(t, manager, account, `{"model":"gpt-6-astra","input":"fixture older independent task"}`, "ctx_pool")) + // A successful plaintext terminal can be delivered without a response + // ID. That delivery must not authorize attachment to another task. + require.NoError(t, manager.write([]byte(`{"type":"response.completed","response":{`+tc.response+`"status":"completed","output":[{"type":"message","role":"assistant","content":"fixture older answer"}]}}`), func() error { return nil })) + current := `{"type":"response.create","model":"gpt-6-astra","input":"fixture new task"` + tc.previous + `}` + cause := openAIWSUnsentTurn(denyOpenAITurn("account_ineligible"), []byte(current), "gpt-6-astra", 2) + got := manager.restoreUnsentHistory(cause) + if !tc.want { + require.Same(t, cause, got, "INDEPENDENT_TASK_ATTACHED: only an explicit nonempty matching completed anchor authorizes history") + body, _, _, ok := OpenAIWSUnsentTurnPayload(got) + require.True(t, ok) + require.JSONEq(t, current, string(body)) + require.NotContains(t, string(body), "fixture older") + if tc.response == "" || tc.response == `"id":"",` || tc.response == `"id":null,` { + require.False(t, manager.localHistory.ready, "a terminal without an anchor cannot retain ready history") + } + return + } + var restored *openAIWSRestoredUnsentError + require.ErrorAs(t, got, &restored) + require.Contains(t, string(restored.body), "fixture older independent task") + require.Contains(t, string(restored.body), "fixture older answer") + require.Contains(t, string(restored.body), "fixture new task") + require.NotContains(t, string(restored.body), "previous_response_id") + require.Same(t, cause, manager.restoreUnsentHistory(cause)) + }) + } +} From f38f99e7280cb5ac57af5bc4e898edab8680b230 Mon Sep 17 00:00:00 2001 From: ranxi2001 Date: Mon, 16 Nov 2026 18:01:23 +0800 Subject: [PATCH 9/9] =?UTF-8?q?fix:=20HTTP=20=E6=A1=A5=E6=8A=95=E5=BD=B1?= =?UTF-8?q?=E5=85=81=E8=AE=B8=E5=8E=9F=E5=A7=8B=E5=B8=A7=E7=9A=84=E5=8F=97?= =?UTF-8?q?=E9=99=90=20ID=20=E8=A7=84=E8=8C=83=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/ws-unsent-delta.yml | 9 ++-- .../handler/openai_ws_unsent_delta_test.go | 48 +++++++++++++++++++ .../internal/service/openai_ws_http_bridge.go | 5 +- 3 files changed, 58 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ws-unsent-delta.yml b/.github/workflows/ws-unsent-delta.yml index 6bd80e350cb1..e3580d4eda1e 100644 --- a/.github/workflows/ws-unsent-delta.yml +++ b/.github/workflows/ws-unsent-delta.yml @@ -5,6 +5,7 @@ on: - 'backend/internal/service/openai_ws_agent_recovery.go' - 'backend/internal/service/openai_ws_forwarder_ingress.go' - 'backend/internal/service/openai_ws_readable_continuation.go' + - 'backend/internal/service/openai_ws_http_bridge.go' - 'backend/internal/handler/openai_ws_unsent_delta_test.go' - 'backend/internal/service/openai_ws_unsent_delta_test.go' - '.github/workflows/ws-unsent-delta.yml' @@ -37,7 +38,7 @@ jobs: - name: Exact baseline implementation for RED if: matrix.phase == 'RED' run: | - for target in backend/internal/service/openai_ws_agent_recovery.go backend/internal/service/openai_ws_forwarder_ingress.go backend/internal/service/openai_ws_readable_continuation.go; do + for target in backend/internal/service/openai_ws_agent_recovery.go backend/internal/service/openai_ws_forwarder_ingress.go backend/internal/service/openai_ws_readable_continuation.go backend/internal/service/openai_ws_http_bridge.go; do git show "2927209f5492d8cb690b83a1964b2f093c19ad7a:$target" > "$target" test "$(git hash-object "$target")" = "$(git rev-parse "2927209f5492d8cb690b83a1964b2f093c19ad7a:$target")" done @@ -48,7 +49,7 @@ jobs: run: | set -euo pipefail set +e - (cd backend && go test -race -json -tags=unit,unsentdelta ./internal/handler -run '^TestOpenAIWSUnsentDelta(Takeover|Fences)$' -count=1 -timeout=5m) > "$RUNNER_TEMP/unsent-delta.jsonl" 2>&1 + (cd backend && go test -race -json -tags=unit,unsentdelta ./internal/handler -run '^TestOpenAIWS(UnsentDelta(Takeover|Fences)|BridgeNormalizedDeltaID)$' -count=1 -timeout=5m) > "$RUNNER_TEMP/unsent-delta.jsonl" 2>&1 result=$? set -e cat "$RUNNER_TEMP/unsent-delta.jsonl" @@ -60,12 +61,14 @@ jobs: outcomes={r.get('Test'):r['Action'] for r in rows if r.get('Action') in ('pass','fail','skip') and r.get('Test')} modes=('http_bridge','ctx_pool','shared','dedicated','passthrough') expected={f'TestOpenAIWSUnsentDeltaTakeover/{m}/turn_{n}' for m in modes for n in (2,53)} + normalized={f'TestOpenAIWSBridgeNormalizedDeltaID/{id}' for id in ('foreign-provider-id','ctc_original_output')} + expected |= normalized fences={f'TestOpenAIWSUnsentDeltaFences/{m}/{f}' for m in ('http_bridge','ctx_pool','passthrough') for f in ('unknown_pointer','unknown_call','terminal_excludes','beta_off')} assert all(outcomes.get(n)=='pass' for n in fences),outcomes if phase=='RED': assert status!=0 and all(outcomes.get(n)=='fail' for n in expected),outcomes for name in expected: - assert 'UNSENT_DELTA_NOT_RESUMED' in ''.join(r.get('Output','') for r in rows if r.get('Test')==name),name + assert ('NORMALIZED_DELTA_REFUSED' if name in normalized else 'UNSENT_DELTA_NOT_RESUMED') in ''.join(r.get('Output','') for r in rows if r.get('Test')==name),name else: assert status==0 and all(outcomes.get(n)=='pass' for n in expected),outcomes assert not any(a in ('fail','skip') for a in outcomes.values()),outcomes diff --git a/backend/internal/handler/openai_ws_unsent_delta_test.go b/backend/internal/handler/openai_ws_unsent_delta_test.go index a06b2ee6e849..92a1e1b492b9 100644 --- a/backend/internal/handler/openai_ws_unsent_delta_test.go +++ b/backend/internal/handler/openai_ws_unsent_delta_test.go @@ -218,3 +218,51 @@ func runUnsentDelta(t *testing.T, mode string, switchTurn int, fence string) { require.Equal(t, "Bearer fixture-B", owners[switchTurn]) require.Equal(t, 1, executions) } + +// Raw custom output IDs use a different prefix from replayable output IDs. +// The source remains immutable; only the existing top-level ID sanitizer may +// explain the difference when the bridge mints a current-turn projection. +func TestOpenAIWSBridgeNormalizedDeltaID(t *testing.T) { + for _, id := range []string{"foreign-provider-id", "ctc_original_output"} { + t.Run(id, func(t *testing.T) { + var mu sync.Mutex + var frames [][]byte + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, e := io.ReadAll(r.Body) + if e != nil { + t.Error(e) + return + } + mu.Lock() + frames = append(frames, body) + n := len(frames) + mu.Unlock() + output := `[{"type":"message","role":"assistant","content":[{"type":"output_text","text":"fixture previous"}]}]` + w.Header().Set("Content-Type", "text/event-stream") + fmt.Fprintf(w, `data: {"type":"response.completed","response":{"id":"resp_normalized_%d","model":"gpt-5.1","status":"completed","output":%s}}`+"\n\n", n, output) + })) + defer upstream.Close() + clients := issue805Redis(t) + headers := http.Header{"Session_id": {"fixture-normalized-session"}, "Thread-Id": {"fixture-normalized-thread"}} + h := newOpenAIWSPassthroughHandlerHarnessWithKey(t, upstream.URL, headers, map[string]string{service.SettingKeyCyberSessionBlockEnabled: "false", service.SettingKeyOpenAI409OmitEncryptedHistoryEnabled: "true"}, repository.NewGatewayCache(clients()), func(k *service.APIKey) { k.AllowLongContext = true }, func(a *service.Account, cfg *config.Config, _ *service.HTTPUpstream, _ *service.AccountRepository) { + a.Extra["openai_apikey_responses_websockets_v2_mode"] = service.OpenAIWSIngressModeHTTPBridge + a.Extra["codex_fingerprint_mode"] = "off" + }) + defer issue600Finish(t, h, headers) + issue715WSSend(t, h.clientConn, `{"type":"response.create","model":"gpt-5.1","store":false,"input":[{"role":"user","content":"fixture original"}]}`) + require.Equal(t, "response.completed", gjson.GetBytes(issue715WSRead(t, h.clientConn), "type").String()) + issue715WSSend(t, h.clientConn, fmt.Sprintf(`{"type":"response.create","model":"gpt-5.1","store":false,"previous_response_id":"resp_normalized_1","input":[{"type":"message","role":"user","id":%q,"content":"fixture next","number":9007199254740993}]}`, id)) + result := issue715WSRead(t, h.clientConn) + require.Equal(t, "response.completed", gjson.GetBytes(result, "type").String(), "NORMALIZED_DELTA_REFUSED: %s", result) + mu.Lock() + defer mu.Unlock() + require.Len(t, frames, 2) + items := gjson.GetBytes(frames[1], "input").Array() + require.Len(t, items, 3) + require.Equal(t, "fixture next", items[2].Get("content").String()) + require.Equal(t, "9007199254740993", items[2].Get("number").Raw) + require.False(t, items[2].Get("id").Exists()) + require.False(t, gjson.GetBytes(frames[1], "previous_response_id").Exists()) + }) + } +} diff --git a/backend/internal/service/openai_ws_http_bridge.go b/backend/internal/service/openai_ws_http_bridge.go index 8c02574442ba..810845ddd78d 100644 --- a/backend/internal/service/openai_ws_http_bridge.go +++ b/backend/internal/service/openai_ws_http_bridge.go @@ -578,6 +578,9 @@ func (s *OpenAIGatewayService) proxyOpenAIWSHTTPBridgeTurn( if account.IsOpenAIApiKey() && s.settingService != nil && s.settingService.IsOpenAI409OmitEncryptedHistoryEnabled(ctx) { source, _ := ctx.Value(openAIWSAgentSourceKey{}).([]byte) previous := gjson.GetBytes(source, "previous_response_id").String() + // The ingress has already stripped invalid item IDs. Derive that one + // permitted normalization from the original frame, never from final wire. + normalizedSource, _, normalizationErr := sanitizeOpenAIResponsesInputItemIDs(source) originalInput, historyInput := gjson.GetBytes(source, "input"), gjson.GetBytes(payload, "input") if previous != "" && gjson.GetBytes(payload, "previous_response_id").String() == previous && gjson.GetBytes(source, "store").Type == gjson.False && !gjson.GetBytes(body, "previous_response_id").Exists() && @@ -591,7 +594,7 @@ func (s *OpenAIGatewayService) proxyOpenAIWSHTTPBridgeTurn( } if len(current) > 0 { encoded, encodeErr := json.Marshal(tail) - if encodeErr == nil && openAIAgentRecoveryInputDigest(source) == openAIAgentRecoveryInputDigest(append(append([]byte(`{"input":`), encoded...), '}')) { + if encodeErr == nil && normalizationErr == nil && openAIAgentRecoveryInputDigest(normalizedSource) == openAIAgentRecoveryInputDigest(append(append([]byte(`{"input":`), encoded...), '}')) { recovery := openAIWSAgentRecoveryFromContext(ctx) if recovery == nil { return nil, openAIAgentRecoveryError("wire_changed")