diff --git a/.github/workflows/ws-unsent-delta.yml b/.github/workflows/ws-unsent-delta.yml new file mode 100644 index 000000000000..5ee0fb3cac41 --- /dev/null +++ b/.github/workflows/ws-unsent-delta.yml @@ -0,0 +1,168 @@ +name: WS unsent delta takeover +on: + pull_request: + paths: + - 'backend/internal/service/openai_ws_agent_recovery.go' + - 'backend/internal/service/openai_ws_forwarder_ingress.go' + - 'backend/internal/service/openai_ws_readable_continuation.go' + - 'backend/internal/service/openai_ws_http_bridge.go' + - 'backend/internal/handler/openai_ws_unsent_delta_test.go' + - 'backend/internal/service/openai_ws_unsent_delta_test.go' + - '.github/workflows/ws-unsent-delta.yml' +permissions: + contents: read +jobs: + takeover: + runs-on: ubuntu-24.04 + timeout-minutes: 25 + strategy: + fail-fast: false + matrix: + phase: [RED, GREEN] + steps: + - uses: actions/checkout@v6 + with: + ref: ${{ github.event.pull_request.head.sha }} + fetch-depth: 0 + persist-credentials: false + - uses: actions/setup-go@v6 + with: + go-version-file: backend/go.mod + cache-dependency-path: backend/go.sum + - name: Redis fixture dependency + run: | + if ! command -v redis-server >/dev/null; then + sudo apt-get update + sudo apt-get install -y --no-install-recommends redis-server + fi + - name: Exact baseline implementation for RED + if: matrix.phase == 'RED' + run: | + for target in backend/internal/service/openai_ws_agent_recovery.go backend/internal/service/openai_ws_forwarder_ingress.go backend/internal/service/openai_ws_readable_continuation.go backend/internal/service/openai_ws_http_bridge.go; do + git show "2927209f5492d8cb690b83a1964b2f093c19ad7a:$target" > "$target" + test "$(git hash-object "$target")" = "$(git rev-parse "2927209f5492d8cb690b83a1964b2f093c19ad7a:$target")" + done + - name: Same socket, account pause, result delta and next turn + env: + PHASE: ${{ matrix.phase }} + shell: bash + run: | + set -euo pipefail + set +e + (cd backend && go test -race -json -tags=unit,unsentdelta ./internal/handler -run '^TestOpenAIWS(UnsentDelta(Takeover|Fences)|BridgeNormalizedDeltaID)$' -count=1 -timeout=5m) > "$RUNNER_TEMP/unsent-delta.jsonl" 2>&1 + result=$? + set -e + cat "$RUNNER_TEMP/unsent-delta.jsonl" + python3 .github/scripts/assert-readable-continuation.py --clean-log "$RUNNER_TEMP/unsent-delta.jsonl" + python3 - "$PHASE" "$result" "$RUNNER_TEMP/unsent-delta.jsonl" <<'PY' + import json,sys + phase,status,path=sys.argv[1],int(sys.argv[2]),sys.argv[3] + rows=[json.loads(l) for l in open(path) if l.startswith('{')] + outcomes={r.get('Test'):r['Action'] for r in rows if r.get('Action') in ('pass','fail','skip') and r.get('Test')} + modes=('http_bridge','ctx_pool','shared','dedicated','passthrough') + expected={f'TestOpenAIWSUnsentDeltaTakeover/{m}/turn_{n}' for m in modes for n in (2,53)} + normalized={f'TestOpenAIWSBridgeNormalizedDeltaID/{id}' for id in ('foreign-provider-id','ctc_original_output')} + expected |= normalized + fences={f'TestOpenAIWSUnsentDeltaFences/{m}/{f}' for m in ('http_bridge','ctx_pool','passthrough') for f in ('unknown_pointer','unknown_call','terminal_excludes','beta_off')} + assert all(outcomes.get(n)=='pass' for n in fences),outcomes + if phase=='RED': + assert status!=0 and all(outcomes.get(n)=='fail' for n in expected),outcomes + for name in expected: + assert ('NORMALIZED_DELTA_REFUSED' if name in normalized else 'UNSENT_DELTA_NOT_RESUMED') in ''.join(r.get('Output','') for r in rows if r.get('Test')==name),name + else: + assert status==0 and all(outcomes.get(n)=='pass' for n in expected),outcomes + assert not any(a in ('fail','skip') for a in outcomes.values()),outcomes + PY + - name: GREEN delivery scope and memory release + if: matrix.phase == 'GREEN' + shell: bash + run: | + set -euo pipefail + (cd backend && go test -race -json -tags=unit,unsentdelta ./internal/service -run '^TestOpenAIWSUnsentDeltaDeliveryScope$' -count=1 -timeout=2m) | tee "$RUNNER_TEMP/unsent-delta-guards.jsonl" + - uses: actions/upload-artifact@v7 + if: always() + with: + name: unsent-delta-${{ matrix.phase }}-${{ github.event.pull_request.head.sha }} + path: ${{ runner.temp }}/unsent-delta*.jsonl + if-no-files-found: error + repair877: + runs-on: ubuntu-24.04 + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + phase: [RED, GREEN] + steps: + - uses: actions/checkout@v6 + with: + ref: ${{ github.event.pull_request.head.sha }} + fetch-depth: 0 + persist-credentials: false + - uses: actions/setup-go@v6 + with: + go-version-file: backend/go.mod + cache-dependency-path: backend/go.sum + - name: Exact reviewed PR877 implementation for repair RED + if: matrix.phase == 'RED' + shell: bash + run: | + set -euo pipefail + # Keep all current tests. Replace only the implementation containing + # the two reviewed faults; the original takeover RED stays separate. + baseline=4b13c785786ff1d623bc23203991dac405494567 + target=backend/internal/service/openai_ws_agent_recovery.go + git show "$baseline:$target" > "$target" + test "$(git hash-object "$target")" = "$(git rev-parse "$baseline:$target")" + - name: Named instruction and continuation-anchor repair evidence + env: + PHASE: ${{ matrix.phase }} + shell: bash + run: | + set -euo pipefail + set +e + (cd backend && go test -race -json -tags=unit,unsentdelta ./internal/service -run '^TestOpenAIWSUnsentDelta(Instructions|ContinuationAnchor|DeliveryScope)$' -count=1 -timeout=3m) > "$RUNNER_TEMP/unsent-delta-repair877.jsonl" 2>&1 + result=$? + set -e + cat "$RUNNER_TEMP/unsent-delta-repair877.jsonl" + python3 .github/scripts/assert-readable-continuation.py --clean-log "$RUNNER_TEMP/unsent-delta-repair877.jsonl" + python3 - "$PHASE" "$result" "$RUNNER_TEMP/unsent-delta-repair877.jsonl" <<'PY' + import json,sys + from collections import Counter + phase,status,path=sys.argv[1],int(sys.argv[2]),sys.argv[3] + rows=[] + for line in open(path): + try: rows.append(json.loads(line)) + except ValueError: pass + package='github.com/Wei-Shaw/sub2api/internal/service' + instructions='TestOpenAIWSUnsentDeltaInstructions' + anchor='TestOpenAIWSUnsentDeltaContinuationAnchor' + delivery='TestOpenAIWSUnsentDeltaDeliveryScope' + roots={instructions,anchor,delivery} + leaves={instructions+'/'+s for s in ('omitted_after_omitted_turn','omitted_after_explicit_turn','current_null','current_explicit','current_empty')} + leaves.update(anchor+'/'+s for s in ('missing_anchor_independent_task','empty_anchor_independent_task','null_anchor_independent_task','valid_anchor_independent_task','valid_anchor_empty_previous','valid_anchor_null_previous','valid_anchor_blank_previous','valid_anchor_wrong_previous','completed_matching_anchor')) + leaves.update(delivery+'/'+s for s in ('valid','pending','failed_write','terminal_excludes','unknown_call','changed_call','wrong_pointer','wrong_key','wrong_model','sent_error','oversize')) + markers={instructions+'/'+s:'STALE_INSTRUCTIONS_RESTORED' for s in ('omitted_after_omitted_turn','omitted_after_explicit_turn')} + markers.update({anchor+'/'+s:'INDEPENDENT_TASK_ATTACHED' for s in ('missing_anchor_independent_task','empty_anchor_independent_task','null_anchor_independent_task')}) + assert phase in ('RED','GREEN'),phase + failures=set(markers)|{instructions,anchor,None} if phase=='RED' else set() + counts=Counter((r.get('Action'),r.get('Test')) for r in rows if r.get('Package')==package) + assert status==(1 if phase=='RED' else 0),status + assert all(r.get('Package')==package for r in rows),rows + assert {r.get('Test') for r in rows if r.get('Action')=='run'}==leaves|roots,rows + assert {r.get('Test') for r in rows if r.get('Action')=='fail'}==failures,rows + assert not any(r.get('Action')=='skip' for r in rows),rows + for name in leaves|roots|{None}: + assert name is None or counts['run',name]==1,(name,counts) + expected='fail' if name in failures else 'pass' + assert counts[expected,name]==1 and counts['pass',name]+counts['fail',name]==1,(name,counts) + if phase=='RED': + for name,marker in markers.items(): + output=''.join(r.get('Output','') for r in rows if r.get('Test')==name) + assert marker in output,(name,output) + PY + - uses: actions/upload-artifact@v7 + if: always() + with: + name: unsent-delta-repair877-${{ matrix.phase }}-${{ github.event.pull_request.head.sha }} + path: ${{ runner.temp }}/unsent-delta-repair877.jsonl + if-no-files-found: error diff --git a/backend/internal/handler/openai_ws_unsent_delta_test.go b/backend/internal/handler/openai_ws_unsent_delta_test.go new file mode 100644 index 000000000000..a216fc50f687 --- /dev/null +++ b/backend/internal/handler/openai_ws_unsent_delta_test.go @@ -0,0 +1,306 @@ +//go:build unit && unsentdelta + +package handler + +import ( + "context" + "encoding/json" + "fmt" + "github.com/Wei-Shaw/sub2api/internal/config" + "github.com/Wei-Shaw/sub2api/internal/repository" + "github.com/Wei-Shaw/sub2api/internal/service" + coderws "github.com/coder/websocket" + "github.com/stretchr/testify/require" + "github.com/tidwall/gjson" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + "time" +) + +func TestOpenAIWSUnsentDeltaTakeover(t *testing.T) { + for _, mode := range readableWSModes { + for _, turn := range []int{2, 53} { + t.Run(fmt.Sprintf("%s/turn_%d", mode, turn), func(t *testing.T) { runUnsentDelta(t, mode, turn, "") }) + } + } +} +func TestOpenAIWSUnsentDeltaFences(t *testing.T) { + for _, mode := range []string{service.OpenAIWSIngressModeHTTPBridge, service.OpenAIWSIngressModeCtxPool, service.OpenAIWSIngressModePassthrough} { + for _, fence := range []string{"unknown_pointer", "unknown_call", "terminal_excludes", "beta_off"} { + t.Run(mode+"/"+fence, func(t *testing.T) { runUnsentDelta(t, mode, 2, fence) }) + } + } +} +func runUnsentDelta(t *testing.T, mode string, switchTurn int, fence string) { + t.Helper() + var mu sync.Mutex + var frames [][]byte + var owners []string + response := func(body []byte, auth string) [][]byte { + mu.Lock() + defer mu.Unlock() + frames = append(frames, append([]byte(nil), body...)) + owners = append(owners, auth) + n := len(frames) + output := fmt.Sprintf(`[{"type":"message","role":"assistant","content":[{"type":"output_text","text":"fixture history %d"}]}]`, n) + if n == switchTurn-1 { + output = `[{"type":"function_call","call_id":"call_takeover_once","name":"fixture","arguments":"{\"n\":9007199254740993}","status":"completed"}]` + } + var events [][]byte + if n == switchTurn-1 && fence == "terminal_excludes" { + // Deliver the real call first. The nonempty terminal is authoritative + // and excludes that call even though the client already received it. + events = append(events, []byte(`{"type":"response.output_item.done","item":`+gjson.Get(output, "0").Raw+`}`)) + output = `[{"type":"message","role":"assistant","content":[{"type":"output_text","text":"fixture terminal excludes delivered tool"}]}]` + } + return append(events, []byte(fmt.Sprintf(`{"type":"response.completed","response":{"id":"resp_delta_%d","model":"gpt-5.1","status":"completed","output":%s,"usage":{"input_tokens":5,"output_tokens":3}}}`, n, output))) + } + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if !strings.EqualFold(r.Header.Get("Upgrade"), "websocket") { + body, e := io.ReadAll(r.Body) + if e != nil { + t.Error(e) + return + } + w.Header().Set("Content-Type", "text/event-stream") + for _, event := range response(body, r.Header.Get("Authorization")) { + fmt.Fprintf(w, "data: %s\n\n", event) + } + return + } + conn, e := coderws.Accept(w, r, nil) + if e != nil { + t.Error(e) + return + } + defer conn.CloseNow() + for { + _, body, e := conn.Read(r.Context()) + if e != nil { + return + } + for _, event := range response(body, r.Header.Get("Authorization")) { + if conn.Write(r.Context(), coderws.MessageText, event) != nil { + return + } + } + } + })) + defer upstream.Close() + clients := issue805Redis(t) + headers := http.Header{"Session_id": {"fixture-delta-session"}, "Thread-Id": {"fixture-delta-thread"}} + var accounts *readableWSAccounts + h := newOpenAIWSPassthroughHandlerHarnessWithKey(t, upstream.URL, headers, + map[string]string{service.SettingKeyCyberSessionBlockEnabled: "false", service.SettingKeyOpenAI409OmitEncryptedHistoryEnabled: fmt.Sprint(fence != "beta_off")}, + repository.NewGatewayCache(clients()), func(k *service.APIKey) { k.AllowLongContext = true }, + func(a *service.Account, cfg *config.Config, _ *service.HTTPUpstream, repo *service.AccountRepository) { + a.ID = 1 + a.Priority = 0 + a.Credentials["api_key"] = "fixture-A" + a.Extra["openai_apikey_responses_websockets_v2_mode"] = mode + a.Extra["codex_fingerprint_mode"] = "off" + cfg.Gateway.MaxAccountSwitches = 2 + cfg.Gateway.Scheduling.OpenAITurnAdmissionCacheTTLSeconds = 0 + cfg.Gateway.OpenAIWS.IngressInterTurnIdleTimeoutSeconds = 20 + b := *a + b.ID = 2 + b.Priority = 1 + b.Credentials = map[string]any{"api_key": "fixture-B", "base_url": upstream.URL} + accounts = &readableWSAccounts{issue600SwitchRepo: &issue600SwitchRepo{&openAIWSFailoverHandlerAccountRepoStub{accounts: []service.Account{*a, b}}}} + *repo = accounts + }) + completion := h.handlerDone + finished := make(chan struct{}) + h.handlerDone = finished + go func() { <-completion; close(finished) }() + defer issue600Finish(t, h, headers) + awaitUsage := func() { + select { + case <-h.usageLogs: + case <-time.After(5 * time.Second): + t.Fatal("completed fixture turn did not settle usage") + } + } + create := func(input, previous string) string { + b := map[string]any{"type": "response.create", "model": "gpt-5.1", "store": true, "input": json.RawMessage(input)} + if previous != "" { + b["previous_response_id"] = previous + } else { + b["tools"] = []any{map[string]any{"type": "function", "name": "fixture", "parameters": map[string]any{"type": "object", "properties": map[string]any{}}}} + } + raw, _ := json.Marshal(b) + return string(raw) + } + issue715WSSend(t, h.clientConn, create(`[{"role":"user","content":"Perform fixture exactly once and retain the task"},{"type":"reasoning","encrypted_content":"fixture-history-state","summary":[{"type":"summary_text","text":"fixture readable progress"}]}]`, "")) + executions := 0 + consume := func(raw []byte) { + items := gjson.GetBytes(raw, "response.output").Array() + if gjson.GetBytes(raw, "type").String() == "response.output_item.done" { + items = append(items, gjson.GetBytes(raw, "item")) + } + for _, item := range items { + if item.Get("type").String() == "function_call" { + require.Equal(t, "call_takeover_once", item.Get("call_id").String()) + executions++ + } + } + } + excludedToolDeliveries := 0 + readTurn := func() []byte { + if fence != "terminal_excludes" { + return issue715WSRead(t, h.clientConn) + } + ctx, cancel := context.WithTimeout(context.Background(), 8*time.Second) + defer cancel() + for { + _, event, err := h.clientConn.Read(ctx) + require.NoError(t, err) + kind := gjson.GetBytes(event, "type").String() + if kind == "response.output_item.done" && gjson.GetBytes(event, "item.type").String() == "function_call" { + require.Equal(t, "call_takeover_once", gjson.GetBytes(event, "item.call_id").String()) + excludedToolDeliveries++ + consume(event) + } + if kind == "response.completed" || kind == "response.failed" || kind == "error" { + return event + } + } + } + first := readTurn() + consume(first) + awaitUsage() + require.Equal(t, "response.completed", gjson.GetBytes(first, "type").String()) + if fence == "terminal_excludes" { + require.Equal(t, 1, excludedToolDeliveries, "terminal exclusion must follow actual tool item delivery to the client") + require.Equal(t, 1, executions) + require.NotContains(t, string(first), "call_takeover_once", "the nonempty terminal must exclude the delivered call") + } + for n := 2; n < switchTurn; n++ { + issue715WSSend(t, h.clientConn, create(fmt.Sprintf(`[{"role":"user","content":"fixture next %d"}]`, n), fmt.Sprintf("resp_delta_%d", n-1))) + round := readTurn() + consume(round) + awaitUsage() + require.Equal(t, "response.completed", gjson.GetBytes(round, "type").String()) + } + accounts.disable(1) + previous := fmt.Sprintf("resp_delta_%d", switchTurn-1) + if fence == "unknown_pointer" { + previous = "resp_foreign" + } + call := "call_takeover_once" + if fence == "unknown_call" { + call = "call_fabricated" + } + issue715WSSend(t, h.clientConn, create(fmt.Sprintf(`[{"type":"function_call_output","call_id":%q,"output":"fixture applied exactly once"}]`, call), previous)) + if fence != "" { + readCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _, raw, e := h.clientConn.Read(readCtx) + if e == nil { + require.Equal(t, "error", gjson.GetBytes(raw, "type").String()) + _, _, e = h.clientConn.Read(readCtx) + } + select { + case <-finished: + case <-readCtx.Done(): + t.Fatal("refusal did not finish; no post-check generation permitted") + } + mu.Lock() + n := len(frames) + mu.Unlock() + require.Equal(t, switchTurn-1, n, "unsafe delta cannot emit another generation") + require.True(t, e != nil || gjson.GetBytes(raw, "type").String() == "error") + return + } + result := issue715WSRead(t, h.clientConn) + consume(result) + if gjson.GetBytes(result, "type").String() != "response.completed" { + logReadableWSRecoveryDetails(t, h) + } + require.Equal(t, "response.completed", gjson.GetBytes(result, "type").String(), "UNSENT_DELTA_NOT_RESUMED: %s", result) + awaitUsage() + mu.Lock() + sent := append([][]byte(nil), frames...) + selected := append([]string(nil), owners...) + mu.Unlock() + require.Len(t, sent, switchTurn) + require.Equal(t, "Bearer fixture-B", selected[switchTurn-1]) + resumed := sent[switchTurn-1] + require.NotContains(t, string(resumed), "resp_delta_", "old provider pointer must not reach B") + require.Contains(t, string(resumed), "Perform fixture exactly once") + require.Contains(t, string(resumed), "fixture applied exactly once") + require.Contains(t, string(resumed), "fixture readable progress") + require.NotContains(t, string(resumed), "fixture-history-state") + require.Contains(t, string(resumed), "9007199254740993") + require.True(t, gjson.GetBytes(resumed, "tools").IsArray(), "effective tool declarations must survive takeover") + for _, item := range gjson.GetBytes(resumed, "input").Array() { + require.NotContains(t, item.Get("type").String(), "call", "completed tools must be inert history") + } + if switchTurn == 53 { + require.Contains(t, string(resumed), "fixture history 1") + require.Contains(t, string(resumed), "fixture next 52") + } + require.Equal(t, 1, executions) + issue715WSSend(t, h.clientConn, create(`[{"role":"user","content":"fixture final followup"}]`, fmt.Sprintf("resp_delta_%d", switchTurn))) + round := issue715WSRead(t, h.clientConn) + consume(round) + awaitUsage() + require.Equal(t, "response.completed", gjson.GetBytes(round, "type").String()) + mu.Lock() + defer mu.Unlock() + require.Len(t, frames, switchTurn+1) + require.Equal(t, "Bearer fixture-B", owners[switchTurn]) + require.Equal(t, 1, executions) +} + +// Raw custom output IDs use a different prefix from replayable output IDs. +// The source remains immutable; only the existing top-level ID sanitizer may +// explain the difference when the bridge mints a current-turn projection. +func TestOpenAIWSBridgeNormalizedDeltaID(t *testing.T) { + for _, id := range []string{"foreign-provider-id", "ctc_original_output"} { + t.Run(id, func(t *testing.T) { + var mu sync.Mutex + var frames [][]byte + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, e := io.ReadAll(r.Body) + if e != nil { + t.Error(e) + return + } + mu.Lock() + frames = append(frames, body) + n := len(frames) + mu.Unlock() + output := `[{"type":"message","role":"assistant","content":[{"type":"output_text","text":"fixture previous"}]}]` + w.Header().Set("Content-Type", "text/event-stream") + fmt.Fprintf(w, `data: {"type":"response.completed","response":{"id":"resp_normalized_%d","model":"gpt-5.1","status":"completed","output":%s}}`+"\n\n", n, output) + })) + defer upstream.Close() + clients := issue805Redis(t) + headers := http.Header{"Session_id": {"fixture-normalized-session"}, "Thread-Id": {"fixture-normalized-thread"}} + h := newOpenAIWSPassthroughHandlerHarnessWithKey(t, upstream.URL, headers, map[string]string{service.SettingKeyCyberSessionBlockEnabled: "false", service.SettingKeyOpenAI409OmitEncryptedHistoryEnabled: "true"}, repository.NewGatewayCache(clients()), func(k *service.APIKey) { k.AllowLongContext = true }, func(a *service.Account, cfg *config.Config, _ *service.HTTPUpstream, _ *service.AccountRepository) { + a.Extra["openai_apikey_responses_websockets_v2_mode"] = service.OpenAIWSIngressModeHTTPBridge + a.Extra["codex_fingerprint_mode"] = "off" + }) + defer issue600Finish(t, h, headers) + issue715WSSend(t, h.clientConn, `{"type":"response.create","model":"gpt-5.1","store":false,"input":[{"role":"user","content":"fixture original"}]}`) + require.Equal(t, "response.completed", gjson.GetBytes(issue715WSRead(t, h.clientConn), "type").String()) + issue715WSSend(t, h.clientConn, fmt.Sprintf(`{"type":"response.create","model":"gpt-5.1","store":false,"previous_response_id":"resp_normalized_1","input":[{"type":"message","role":"user","id":%q,"content":"fixture next","number":9007199254740993}]}`, id)) + result := issue715WSRead(t, h.clientConn) + require.Equal(t, "response.completed", gjson.GetBytes(result, "type").String(), "NORMALIZED_DELTA_REFUSED: %s", result) + mu.Lock() + defer mu.Unlock() + require.Len(t, frames, 2) + items := gjson.GetBytes(frames[1], "input").Array() + require.Len(t, items, 3) + require.Equal(t, "fixture next", items[2].Get("content").String()) + require.Equal(t, "9007199254740993", items[2].Get("number").Raw) + require.False(t, items[2].Get("id").Exists()) + require.False(t, gjson.GetBytes(frames[1], "previous_response_id").Exists()) + }) + } +} diff --git a/backend/internal/service/openai_ws_agent_recovery.go b/backend/internal/service/openai_ws_agent_recovery.go index a68528173054..fd7d062e250c 100644 --- a/backend/internal/service/openai_ws_agent_recovery.go +++ b/backend/internal/service/openai_ws_agent_recovery.go @@ -4,17 +4,21 @@ import ( "bytes" "context" "encoding/json" + "errors" "io" "net/http" "net/url" + "strings" "sync" "sync/atomic" "time" + "unicode/utf8" "github.com/Wei-Shaw/sub2api/internal/pkg/logger" coderws "github.com/coder/websocket" "github.com/gin-gonic/gin" "github.com/tidwall/gjson" + "github.com/tidwall/sjson" "go.uber.org/zap" ) @@ -63,6 +67,8 @@ type openAIWSAgentRecovery struct { completedBridge openAIWSBridgeResponseAnchor deliveredTools map[string]openAIWSDeliveredTool pendingTools map[string]openAIWSDeliveredTool + localHistory openAIWSLocalHistory + historyReserved int64 } func newOpenAIWSAgentRecovery(s *OpenAIGatewayService, ctx context.Context, c *gin.Context) *openAIWSAgentRecovery { @@ -89,6 +95,7 @@ func (r *openAIWSAgentRecovery) abort() { } r.mu.Lock() defer r.mu.Unlock() + defer func() { r.localHistory = openAIWSLocalHistory{}; r.accountLocalHistory() }() if r.finish != nil { r.finish(false) r.finish = nil @@ -407,6 +414,7 @@ func (r *openAIWSAgentRecovery) begin(account *Account, source, payload []byte, return err } r.finish = finish + r.stageLocalHistory(source) if r.request.owner != nil || len(r.request.items) > 0 || r.request.omittedHistory || (r.service.settingService != nil && r.service.settingService.IsOpenAI409OmitEncryptedHistoryEnabled(ctx)) { // In automatic continuation mode, even a plaintext initial generation @@ -460,6 +468,16 @@ func (r *openAIWSAgentRecovery) write(payload []byte, write func() error) error r.finish = nil return err } + if r.localHistory.pending != nil { + r.localHistory.outputBytes += len(payload) + if r.localHistory.outputBytes > openAIWSLocalHistoryLimit { + r.localHistory = openAIWSLocalHistory{} + r.accountLocalHistory() + } else { + r.localHistory.output.AddEvent(event, payload) + r.accountLocalHistory() + } + } toolItems := gjson.GetBytes(payload, "response.output").Array() if event == "response.output_item.done" { toolItems = append(toolItems, gjson.GetBytes(payload, "item")) @@ -555,6 +573,7 @@ func (r *openAIWSAgentRecovery) write(payload []byte, write func() error) error r.known[request.itemKey(item)] = true } } + r.commitLocalHistory(completed) r.finish = nil } return nil @@ -588,7 +607,26 @@ func openAIWSWireRefusal(ctx context.Context, stage string, source, payload []by storeKind = "other" } } + sanitized, idsChanged, sanitizeErr := sanitizeOpenAIResponsesInputItemIDs(source) + tailMatchesSource, tailMatchesIDs := false, false + if sourceInput.IsArray() && wireInput.IsArray() { + original, wire := sourceInput.Array(), wireInput.Array() + if len(original) > 0 && len(wire) >= len(original) { + var tail []json.RawMessage + for _, item := range wire[len(wire)-len(original):] { + tail = append(tail, json.RawMessage(item.Raw)) + } + encoded, e := json.Marshal(tail) + if e == nil { + tailBody := append(append([]byte(`{"input":`), encoded...), '}') + tailMatchesSource = openAIAgentRecoveryInputDigest(source) == openAIAgentRecoveryInputDigest(tailBody) + tailMatchesIDs = sanitizeErr == nil && openAIAgentRecoveryInputDigest(sanitized) == openAIAgentRecoveryInputDigest(tailBody) + } + } + } logger.FromContext(ctx).Warn("openai.encrypted_history_409_refused", zap.String("stage", stage), + zap.Bool("source_ids_normalized", idsChanged), zap.Bool("tail_matches_source", tailMatchesSource), zap.Bool("tail_matches_normalized_ids", tailMatchesIDs), + zap.Int("source_agent_items", len(openAIAgentRecoveryItemDigests(source))), zap.Int("wire_agent_items", len(openAIAgentRecoveryItemDigests(payload))), zap.String("source_input_kind", inputKind), zap.String("source_store", storeKind), zap.Int("source_input_items", len(sourceInput.Array())), zap.Int("wire_input_items", len(wireInput.Array())), zap.Bool("source_previous_present", gjson.GetBytes(source, "previous_response_id").String() != ""), @@ -598,3 +636,364 @@ func openAIWSWireRefusal(ctx context.Context, stage string, source, payload []by zap.Bool("controls_changed", openAI409ControlsDigest(source) != openAI409ControlsDigest(payload))) return openAIRecoveryDiagnosticError("wire_changed", stage, openAIAgentRecoveryFromContext(ctx)) } + +// Bounded, manager-local plaintext history for an explicitly unsent next turn. +// It expires with the socket/forwarding attempt, never enters Redis or logs. +const openAIWSLocalHistoryLimit = 8 << 20 + +var openAIWSLocalHistoryBytes atomic.Int64 + +type openAIWSLocalHistory struct { + outputBytes int + tools json.RawMessage + committed []json.RawMessage + pending []json.RawMessage + output openAIWSLocalOutput + anchor openAIWSBridgeResponseAnchor + clientModel string + ready bool +} + +type openAIWSRestoredUnsentError struct{ *openAIWSUnsentTurnError } + +func (e *openAIWSRestoredUnsentError) Unwrap() error { return e.openAIWSUnsentTurnError } +func (e *openAIWSRestoredUnsentError) RecoveredWSContext(ctx context.Context) context.Context { + return WithRecoveredContextPrivacy(ctx) +} + +func openAIWSPlainItems(body []byte) ([]json.RawMessage, bool) { + if len(body) > openAIWSLocalHistoryLimit { + return nil, false + } + if !utf8.Valid(body) || !openAIEncryptedJSONUnambiguous(body) { + return nil, false + } + // Historical reasoning can contain a readable summary plus provider state. + // Apply the existing explicit Beta representation, preserving supplied text + // and marking unavailable state. Tool data and current agent envelopes are + // still rejected below; this never fabricates a task from ciphertext. + hasReasoning := false + for _, item := range gjson.GetBytes(body, "input").Array() { + hasReasoning = hasReasoning || item.Get("type").String() == "reasoning" + } + if hasReasoning { + var e error + body, e = prepareOpenAI409ReadableReplay(body) + if e != nil || len(body) > openAIWSLocalHistoryLimit { + return nil, false + } + } + input := gjson.GetBytes(body, "input") + var items []json.RawMessage + if input.Type == gjson.String { + raw, _ := json.Marshal(map[string]any{"type": "message", "role": "user", "content": input.String()}) + items = []json.RawMessage{raw} + } else if input.IsArray() { + for _, item := range input.Array() { + items = append(items, json.RawMessage(item.Raw)) + } + } else { + return nil, false + } + size := 0 + for _, raw := range items { + size += len(raw) + if size > openAIWSLocalHistoryLimit || !openAIWSPlainItem(raw) { + return nil, false + } + } + return items, true +} +func openAIWSPlainItem(raw []byte) bool { + item := gjson.ParseBytes(raw) + if !item.IsObject() || openAIWSHasOpaqueField(item, 0) { + return false + } + switch item.Get("type").String() { + case "", "message": + if role := item.Get("role").String(); role != "user" && role != "assistant" && role != "system" && role != "developer" { + return false + } + return openAIReadableShadowTextOnly(item.Get("content")) + case "function_call", "custom_tool_call": + _, ok := openAIToolFingerprint(item) + return ok + case "function_call_output", "custom_tool_call_output": + return item.Get("call_id").Type == gjson.String && item.Get("call_id").String() != "" && openAIReadableShadowTextOnly(item.Get("output")) + default: + return false + } +} +func openAIWSHistoryCopy(items []json.RawMessage) ([]json.RawMessage, bool) { + size := 0 + for _, v := range items { + size += len(v) + if size > openAIWSLocalHistoryLimit { + return nil, false + } + } + out := make([]json.RawMessage, len(items)) + for i, v := range items { + out[i] = append(json.RawMessage(nil), v...) + } + return out, true +} +func (r *openAIWSAgentRecovery) stageLocalHistory(source []byte) { + defer r.accountLocalHistory() + r.localHistory.pending = nil + r.localHistory.outputBytes = 0 + r.localHistory.output = openAIWSLocalOutput{} + if r.service.settingService == nil || !r.service.settingService.IsOpenAI409OmitEncryptedHistoryEnabled(r.ctx) { + r.localHistory = openAIWSLocalHistory{} + return + } + items, ok := openAIWSPlainItems(source) + if !ok || gjson.GetBytes(source, "conversation").Exists() { + r.localHistory.ready = false + return + } + previous := gjson.GetBytes(source, "previous_response_id").String() + if previous != "" { + anchor := r.localHistory.anchor + if !r.localHistory.ready || previous != anchor.response || r.request.keyID != anchor.key || r.request.userID != anchor.user || r.request.groupID != anchor.group || r.request.session != anchor.session || r.request.model != anchor.model { + r.localHistory.ready = false + return + } + // Accept either a delta or the exact complete prefix; never drop existing messages. + if !openAIWSRawItemsHasPrefix(items, r.localHistory.committed) { + items = combineOpenAIWSReplayItems(r.localHistory.committed, items) + } + } + if previous == "" { + r.localHistory.tools = nil + } + // Tools retain the socket's declarations. Instructions apply only to the + // current frame, including when omitted, and must never enter this history. + if value := gjson.GetBytes(source, "tools"); value.Exists() { + r.localHistory.tools = append(json.RawMessage(nil), value.Raw...) + } + if len(r.localHistory.tools) > openAIWSLocalHistoryLimit { + r.localHistory = openAIWSLocalHistory{} + return + } + r.localHistory.pending, ok = openAIWSHistoryCopy(items) + r.localHistory.ready = false + if !ok { + r.localHistory.pending = nil + } + if model := gjson.GetBytes(source, "model").String(); model != "" { + r.localHistory.clientModel = model + } +} +func (r *openAIWSAgentRecovery) commitLocalHistory(completed bool) { + defer r.accountLocalHistory() + h := &r.localHistory + if !completed || h.pending == nil || strings.TrimSpace(r.completedBridge.response) == "" { + h.ready = false + h.committed = nil + return + } + output := h.output.items + if h.output.final != nil { + output = h.output.final + } + if output == nil { + output = []json.RawMessage{} + } + encoded, e := json.Marshal(output) + if e != nil { + h.ready = false + h.committed = nil + return + } + normalized, ok := openAIWSPlainItems(append(append([]byte(`{"input":`), encoded...), '}')) + if !ok { + h.ready = false + h.committed = nil + return + } + output = normalized + all := combineOpenAIWSReplayItems(h.pending, output) + h.committed, ok = openAIWSHistoryCopy(all) + h.ready = ok + if ok { + h.anchor = r.completedBridge + } + h.pending = nil + h.outputBytes = 0 + h.output = openAIWSLocalOutput{} +} + +// Called at the outer proxy return boundary, after transport goroutines stop. +// Only the typed not-written admission error grants permission to attach prior +// delivered context. Every current auth/moderation/long-context gate runs again. +func (r *openAIWSAgentRecovery) restoreUnsentHistory(err error) error { + if r == nil { + return err + } + var unsent *openAIWSUnsentTurnError + var admission *OpenAITurnAdmissionError + if !errors.As(err, &unsent) || unsent.keepAccount || !errors.As(err, &admission) { + return err + } + r.mu.Lock() + defer r.mu.Unlock() + h := &r.localHistory + if !h.ready || r.finish != nil || r.service.settingService == nil || !r.service.settingService.IsOpenAI409OmitEncryptedHistoryEnabled(r.ctx) { + return err + } + source := unsent.body + previous := gjson.GetBytes(source, "previous_response_id") + // Two blank IDs describe no continuation. Only the client's explicit + // pointer to a delivered, completed response may attach another turn. + if len(source) > openAIWSLocalHistoryLimit || previous.Type != gjson.String || strings.TrimSpace(previous.String()) == "" || + strings.TrimSpace(h.anchor.response) == "" || previous.String() != h.anchor.response || gjson.GetBytes(source, "conversation").Exists() { + return err + } + if unsent.model != "" && unsent.model != h.clientModel { + return err + } + scope := openAIAgentRecoveryFromContext(r.service.WithOpenAIAgentMessageRecovery(r.ctx, r.c, source, r.session, h.anchor.model)) + if scope.userID <= 0 || scope.keyID <= 0 || scope.userID != h.anchor.user || scope.keyID != h.anchor.key || scope.groupID != h.anchor.group || scope.session != h.anchor.session || scope.lookupErr != nil || scope.conflict || scope.disallowLearning { + return err + } + delta, ok := openAIWSPlainItems(source) + if !ok { + return err + } + all := delta + if !openAIWSRawItemsHasPrefix(delta, h.committed) { + all = combineOpenAIWSReplayItems(h.committed, delta) + } + all, ok = openAIWSHistoryCopy(all) + if !ok { + return err + } + expanded, e := setOpenAIWSPayloadInputSequence(source, all, true) + if e != nil { + return err + } + if _, why := inspectOpenAIWSCompletedTools(expanded); why != "" { + return err + } + // Every executed historical call must be one actually delivered on this + // manager with a successful terminal verdict, including encrypted anchors. + for _, item := range gjson.GetBytes(expanded, "input").Array() { + kind := item.Get("type").String() + if kind != "function_call" && kind != "custom_tool_call" { + continue + } + fp, valid := openAIToolFingerprint(item) + proof, found := r.deliveredTools[h.anchor.model+"\x00"+item.Get("call_id").String()] + if !valid || !found || proof.conflicted || proof.fingerprint != fp || proof.accountID != h.anchor.account || proof.config != h.anchor.config { + return err + } + } + if !gjson.GetBytes(expanded, "tools").Exists() && len(h.tools) > 0 { + expanded, e = sjson.SetRawBytes(expanded, "tools", h.tools) + if e != nil { + return err + } + } + if !openAI409HasReadableIntent(expanded) { + return err + } + // Completed calls become inert history, so a replacement provider cannot + // execute the original tool call again. Keep arguments/results as exact JSON. + expanded, e = projectOpenAIWSCompletedTools(expanded) + if e != nil { + return err + } + expanded, e = sjson.DeleteBytes(expanded, "previous_response_id") + if e != nil || len(expanded) > openAIWSLocalHistoryLimit { + return err + } + copy := *unsent + copy.body = expanded + h.ready = false // proof consumed once; no reuse after any downstream failure + return &openAIWSRestoredUnsentError{©} +} + +// Budget retained context across all sockets; exhaustion only disables this +// optional restoration path. It never rejects normal forwarding or drops bytes. +func (r *openAIWSAgentRecovery) accountLocalHistory() { + h := &r.localHistory + wanted := int64(len(h.tools) + 3*h.outputBytes) + for _, set := range [][]json.RawMessage{h.pending, h.committed} { + for _, v := range set { + wanted += int64(len(v)) + } + } + delta := wanted - r.historyReserved + if delta > 0 { + for { + old := openAIWSLocalHistoryBytes.Load() + if old+delta > 128<<20 { + r.localHistory = openAIWSLocalHistory{} + openAIWSLocalHistoryBytes.Add(-r.historyReserved) + r.historyReserved = 0 + return + } + if openAIWSLocalHistoryBytes.CompareAndSwap(old, old+delta) { + break + } + } + } else { + openAIWSLocalHistoryBytes.Add(delta) + } + r.historyReserved = wanted +} + +func openAIWSHasOpaqueField(value gjson.Result, depth int) bool { + if depth > 64 { + return true + } + if !value.IsObject() && !value.IsArray() { + return false + } + opaque := false + value.ForEach(func(key, child gjson.Result) bool { + opaque = key.String() == "encrypted_content" || openAIWSHasOpaqueField(child, depth+1) + return !opaque + }) + return opaque +} + +// Kept with the manager: protocol regression baselines may restore an older +// bridge collector, while this socket history still needs terminal authority. +type openAIWSLocalOutput struct { + items, final []json.RawMessage + seen map[string]bool +} + +func (o *openAIWSLocalOutput) AddEvent(event string, payload []byte) { + var items []gjson.Result + if event == "response.output_item.done" { + items = []gjson.Result{gjson.GetBytes(payload, "item")} + } + if (event == "response.completed" || event == "response.done") && openAIAgentCompletedResponse(payload, true) { + items = gjson.GetBytes(payload, "response.output").Array() + if len(items) > 0 { + o.final = []json.RawMessage{} + for _, v := range items { + o.final = append(o.final, append(json.RawMessage(nil), v.Raw...)) + } + } + } + for _, v := range items { + key := v.Get("id").String() + if key == "" { + key = v.Get("call_id").String() + } + if key == "" { + key = v.Raw + } + if o.seen == nil { + o.seen = map[string]bool{} + } + if !o.seen[key] { + o.seen[key] = true + o.items = append(o.items, append(json.RawMessage(nil), v.Raw...)) + } + } +} diff --git a/backend/internal/service/openai_ws_forwarder_ingress.go b/backend/internal/service/openai_ws_forwarder_ingress.go index 24dc2661ca5e..32c94fe8e8ba 100644 --- a/backend/internal/service/openai_ws_forwarder_ingress.go +++ b/backend/internal/service/openai_ws_forwarder_ingress.go @@ -589,6 +589,11 @@ func (s *OpenAIGatewayService) ProxyResponsesWebSocketFromClient( agentRecovery := newOpenAIWSAgentRecovery(s, ctx, c) ctx = context.WithValue(ctx, openAIWSAgentRecoveryKey{}, agentRecovery) defer agentRecovery.abort() + defer func() { + if recovery, ok := any(agentRecovery).(interface{ restoreUnsentHistory(error) error }); ok { + returnErr = recovery.restoreUnsentHistory(returnErr) + } + }() var meterCleanup func() var meterErr error ctx, hooks, meterCleanup, meterErr = s.wrapRegionalWSTurns(ctx, c, account, firstClientMessage, hooks) diff --git a/backend/internal/service/openai_ws_http_bridge.go b/backend/internal/service/openai_ws_http_bridge.go index 8c02574442ba..810845ddd78d 100644 --- a/backend/internal/service/openai_ws_http_bridge.go +++ b/backend/internal/service/openai_ws_http_bridge.go @@ -578,6 +578,9 @@ func (s *OpenAIGatewayService) proxyOpenAIWSHTTPBridgeTurn( if account.IsOpenAIApiKey() && s.settingService != nil && s.settingService.IsOpenAI409OmitEncryptedHistoryEnabled(ctx) { source, _ := ctx.Value(openAIWSAgentSourceKey{}).([]byte) previous := gjson.GetBytes(source, "previous_response_id").String() + // The ingress has already stripped invalid item IDs. Derive that one + // permitted normalization from the original frame, never from final wire. + normalizedSource, _, normalizationErr := sanitizeOpenAIResponsesInputItemIDs(source) originalInput, historyInput := gjson.GetBytes(source, "input"), gjson.GetBytes(payload, "input") if previous != "" && gjson.GetBytes(payload, "previous_response_id").String() == previous && gjson.GetBytes(source, "store").Type == gjson.False && !gjson.GetBytes(body, "previous_response_id").Exists() && @@ -591,7 +594,7 @@ func (s *OpenAIGatewayService) proxyOpenAIWSHTTPBridgeTurn( } if len(current) > 0 { encoded, encodeErr := json.Marshal(tail) - if encodeErr == nil && openAIAgentRecoveryInputDigest(source) == openAIAgentRecoveryInputDigest(append(append([]byte(`{"input":`), encoded...), '}')) { + if encodeErr == nil && normalizationErr == nil && openAIAgentRecoveryInputDigest(normalizedSource) == openAIAgentRecoveryInputDigest(append(append([]byte(`{"input":`), encoded...), '}')) { recovery := openAIWSAgentRecoveryFromContext(ctx) if recovery == nil { return nil, openAIAgentRecoveryError("wire_changed") diff --git a/backend/internal/service/openai_ws_readable_continuation.go b/backend/internal/service/openai_ws_readable_continuation.go index 355fb825f7bd..1a966587f726 100644 --- a/backend/internal/service/openai_ws_readable_continuation.go +++ b/backend/internal/service/openai_ws_readable_continuation.go @@ -103,6 +103,12 @@ type openAIWSUnsentOwnerProofKey struct{} // Carry the route check's observations through the handler's auth boundary. // Rebuild consumes this current-turn evidence once; it cannot leak to later turns. func WithOpenAIWSUnsentOwnerProof(ctx context.Context, err error) context.Context { + var recovered interface { + RecoveredWSContext(context.Context) context.Context + } + if errors.As(err, &recovered) { + ctx = recovered.RecoveredWSContext(ctx) + } var unsent *openAIWSUnsentTurnError if errors.As(err, &unsent) && unsent.completedSource != nil { return context.WithValue(ctx, openAIWSUnsentOwnerProofKey{}, unsent.completedSource) diff --git a/backend/internal/service/openai_ws_unsent_delta_test.go b/backend/internal/service/openai_ws_unsent_delta_test.go new file mode 100644 index 000000000000..3ca371c2655b --- /dev/null +++ b/backend/internal/service/openai_ws_unsent_delta_test.go @@ -0,0 +1,188 @@ +//go:build unsentdelta + +package service + +import ( + "context" + "errors" + "strings" + "testing" + + "github.com/Wei-Shaw/sub2api/internal/config" + "github.com/stretchr/testify/require" + "github.com/tidwall/gjson" +) + +func TestOpenAIWSUnsentDeltaDeliveryScope(t *testing.T) { + for _, mutation := range []string{"valid", "pending", "failed_write", "terminal_excludes", "unknown_call", "changed_call", "wrong_pointer", "wrong_key", "wrong_model", "sent_error", "oversize"} { + t.Run(mutation, func(t *testing.T) { + before := openAIWSLocalHistoryBytes.Load() + svc := &OpenAIGatewayService{cfg: &config.Config{}, settingService: NewSettingService(&codexPolicyMigrationRepoStub{values: map[string]string{SettingKeyOpenAI409OmitEncryptedHistoryEnabled: "true"}}, nil)} + manager, c, a := issue715WSManager(t, svc, context.Background()) + SetOpenAIWSRecoveryPrincipal(c, 715, 715) + defer func() { + manager.abort() + require.Equal(t, before, openAIWSLocalHistoryBytes.Load(), "socket teardown must release retained history") + }() + source := `{"model":"gpt-6-astra","input":[{"role":"user","content":"fixture original task"}]}` + require.NoError(t, issue715WSBegin(t, manager, a, source, "ctx_pool")) + call := `{"type":"function_call","call_id":"call_fixture_delta","name":"fixture","arguments":"{}","status":"completed"}` + if mutation == "pending" || mutation == "terminal_excludes" { + delivered := false + require.NoError(t, manager.write([]byte(`{"type":"response.output_item.done","item":`+call+`}`), func() error { + delivered = true + return nil + })) + require.True(t, delivered, "the tool item must reach the client before its terminal verdict") + } + if mutation != "pending" { + output := call + if mutation == "terminal_excludes" { + output = `{"type":"message","role":"assistant","content":"fixture complete without tool"}` + } + err := manager.write([]byte(`{"type":"response.completed","response":{"id":"resp_fixture_delta","status":"completed","output":[`+output+`]}}`), func() error { + if mutation == "failed_write" { + return errors.New("fixture write failed") + } + return nil + }) + if mutation == "failed_write" { + require.Error(t, err) + } else { + require.NoError(t, err) + } + } + current := `{"type":"response.create","model":"gpt-6-astra","previous_response_id":"resp_fixture_delta","input":[{"type":"function_call_output","call_id":"call_fixture_delta","output":"fixture executed once"}]}` + model := "gpt-6-astra" + switch mutation { + case "unknown_call": + current = strings.Replace(current, "call_fixture_delta", "call_unknown", 1) + case "changed_call": + manager.deliveredTools[model+"\x00call_fixture_delta"] = openAIWSDeliveredTool{fingerprint: "wrong", accountID: a.ID} + case "wrong_pointer": + current = strings.Replace(current, "resp_fixture_delta", "resp_foreign", 1) + case "wrong_key": + c.Set("api_key", &APIKey{ID: 999, GroupID: int64Ptr(715)}) + case "wrong_model": + model = "gpt-6-sol" + case "oversize": + current = strings.Replace(current, "fixture executed once", strings.Repeat("x", openAIWSLocalHistoryLimit+1), 1) + } + cause := openAIWSUnsentTurn(denyOpenAITurn("account_ineligible"), []byte(current), model, 2) + if mutation == "sent_error" { + cause = errors.New("fixture unknown upstream delivery") + } + got := manager.restoreUnsentHistory(cause) + var restored *openAIWSRestoredUnsentError + if mutation == "valid" { + require.ErrorAs(t, got, &restored) + require.Contains(t, string(restored.body), "fixture original task") + require.Contains(t, string(restored.body), "fixture executed once") + require.NotContains(t, string(restored.body), `"type":"function_call"`) + require.NotContains(t, string(restored.body), "previous_response_id") + require.Same(t, cause, manager.restoreUnsentHistory(cause), "proof is consumed once") + } else { + require.Same(t, cause, got, "no evidence may be fabricated from failed delivery or foreign scope") + } + }) + } +} + +func TestOpenAIWSUnsentDeltaInstructions(t *testing.T) { + for _, tc := range []struct { + name string + second string + third string + }{ + {"omitted_after_omitted_turn", "", ""}, + {"omitted_after_explicit_turn", `,"instructions":"fixture turn two only"`, ""}, + {"current_null", "", `,"instructions":null`}, + {"current_explicit", "", `,"instructions":"fixture current control"`}, + {"current_empty", "", `,"instructions":""`}, + } { + t.Run(tc.name, func(t *testing.T) { + before := openAIWSLocalHistoryBytes.Load() + svc := &OpenAIGatewayService{cfg: &config.Config{}, settingService: NewSettingService(&codexPolicyMigrationRepoStub{values: map[string]string{SettingKeyOpenAI409OmitEncryptedHistoryEnabled: "true"}}, nil)} + manager, c, account := issue715WSManager(t, svc, context.Background()) + SetOpenAIWSRecoveryPrincipal(c, 715, 715) + defer func() { + manager.abort() + require.Equal(t, before, openAIWSLocalHistoryBytes.Load()) + }() + first := `{"model":"gpt-6-astra","instructions":"fixture turn one only","tools":[{"type":"function","name":"fixture","parameters":{"type":"object"}}],"input":"fixture original task"}` + require.NoError(t, issue715WSBegin(t, manager, account, first, "ctx_pool")) + require.NoError(t, manager.write([]byte(`{"type":"response.completed","response":{"id":"resp_instructions_1","status":"completed","output":[{"type":"message","role":"assistant","content":"fixture first answer"}]}}`), func() error { return nil })) + second := `{"model":"gpt-6-astra","previous_response_id":"resp_instructions_1","input":"fixture second task"` + tc.second + `}` + require.NoError(t, issue715WSBegin(t, manager, account, second, "ctx_pool")) + require.NoError(t, manager.write([]byte(`{"type":"response.completed","response":{"id":"resp_instructions_2","status":"completed","output":[{"type":"message","role":"assistant","content":"fixture second answer"}]}}`), func() error { return nil })) + third := `{"type":"response.create","model":"gpt-6-astra","previous_response_id":"resp_instructions_2","input":"fixture current task"` + tc.third + `}` + cause := openAIWSUnsentTurn(denyOpenAITurn("account_ineligible"), []byte(third), "gpt-6-astra", 3) + var restored *openAIWSRestoredUnsentError + require.ErrorAs(t, manager.restoreUnsentHistory(cause), &restored) + want, got := gjson.Get(third, "instructions"), gjson.GetBytes(restored.body, "instructions") + require.Equal(t, want.Exists(), got.Exists(), "STALE_INSTRUCTIONS_RESTORED: preserve current-frame field presence") + require.Equal(t, want.Raw, got.Raw, "STALE_INSTRUCTIONS_RESTORED: preserve current-frame null or explicit value") + require.Equal(t, gjson.Get(first, "tools").Raw, gjson.GetBytes(restored.body, "tools").Raw, "socket tool declarations still survive takeover") + for _, text := range []string{"fixture original task", "fixture first answer", "fixture second task", "fixture second answer", "fixture current task"} { + require.Contains(t, string(restored.body), text) + } + require.NotContains(t, string(restored.body), "previous_response_id") + require.Same(t, cause, manager.restoreUnsentHistory(cause), "restoration proof is consumed once") + }) + } +} + +func TestOpenAIWSUnsentDeltaContinuationAnchor(t *testing.T) { + for _, tc := range []struct { + name string + response string + previous string + want bool + }{ + {"missing_anchor_independent_task", "", "", false}, + {"empty_anchor_independent_task", `"id":"",`, `,"previous_response_id":""`, false}, + {"null_anchor_independent_task", `"id":null,`, `,"previous_response_id":null`, false}, + {"valid_anchor_independent_task", `"id":"resp_anchor",`, "", false}, + {"valid_anchor_empty_previous", `"id":"resp_anchor",`, `,"previous_response_id":""`, false}, + {"valid_anchor_null_previous", `"id":"resp_anchor",`, `,"previous_response_id":null`, false}, + {"valid_anchor_blank_previous", `"id":"resp_anchor",`, `,"previous_response_id":" "`, false}, + {"valid_anchor_wrong_previous", `"id":"resp_anchor",`, `,"previous_response_id":"resp_foreign"`, false}, + {"completed_matching_anchor", `"id":"resp_anchor",`, `,"previous_response_id":"resp_anchor"`, true}, + } { + t.Run(tc.name, func(t *testing.T) { + before := openAIWSLocalHistoryBytes.Load() + svc := &OpenAIGatewayService{cfg: &config.Config{}, settingService: NewSettingService(&codexPolicyMigrationRepoStub{values: map[string]string{SettingKeyOpenAI409OmitEncryptedHistoryEnabled: "true"}}, nil)} + manager, c, account := issue715WSManager(t, svc, context.Background()) + SetOpenAIWSRecoveryPrincipal(c, 715, 715) + defer func() { + manager.abort() + require.Equal(t, before, openAIWSLocalHistoryBytes.Load()) + }() + require.NoError(t, issue715WSBegin(t, manager, account, `{"model":"gpt-6-astra","input":"fixture older independent task"}`, "ctx_pool")) + // A successful plaintext terminal can be delivered without a response + // ID. That delivery must not authorize attachment to another task. + require.NoError(t, manager.write([]byte(`{"type":"response.completed","response":{`+tc.response+`"status":"completed","output":[{"type":"message","role":"assistant","content":"fixture older answer"}]}}`), func() error { return nil })) + current := `{"type":"response.create","model":"gpt-6-astra","input":"fixture new task"` + tc.previous + `}` + cause := openAIWSUnsentTurn(denyOpenAITurn("account_ineligible"), []byte(current), "gpt-6-astra", 2) + got := manager.restoreUnsentHistory(cause) + if !tc.want { + require.Same(t, cause, got, "INDEPENDENT_TASK_ATTACHED: only an explicit nonempty matching completed anchor authorizes history") + body, _, _, ok := OpenAIWSUnsentTurnPayload(got) + require.True(t, ok) + require.JSONEq(t, current, string(body)) + require.NotContains(t, string(body), "fixture older") + if tc.response == "" || tc.response == `"id":"",` || tc.response == `"id":null,` { + require.False(t, manager.localHistory.ready, "a terminal without an anchor cannot retain ready history") + } + return + } + var restored *openAIWSRestoredUnsentError + require.ErrorAs(t, got, &restored) + require.Contains(t, string(restored.body), "fixture older independent task") + require.Contains(t, string(restored.body), "fixture older answer") + require.Contains(t, string(restored.body), "fixture new task") + require.NotContains(t, string(restored.body), "previous_response_id") + require.Same(t, cause, manager.restoreUnsentHistory(cause)) + }) + } +}