diff --git a/.github/scripts/prepare-replay-red-fixtures.sh b/.github/scripts/prepare-replay-red-fixtures.sh index 72a45caa89d0..f35e8e48b267 100644 --- a/.github/scripts/prepare-replay-red-fixtures.sh +++ b/.github/scripts/prepare-replay-red-fixtures.sh @@ -44,6 +44,30 @@ if ! test -f "$tree/$seam"; then exclude_absent_fixture backend/internal/handler/openai_issue842_oauth_repair_test.go backend/internal/service/openai_ws_agent_recovery.go fi +# PR878 added a Group field used by otherwise unrelated service-package tests. +# A full old-source overlay must compile them against that same historical API; +# selected recovery oracles and all candidate GREEN tests remain untouched. +contract=backend/internal/service/group.go +if ! git -C "$tree" grep -q 'FreeOpenAIUltrafast' "$baseline" -- "$contract"; then + expected_contract=$(git -C "$tree" rev-parse "$baseline:$contract") + test "$(git -C "$tree" hash-object "$contract")" = "$expected_contract" + for target in backend/internal/service/admin_group_duplicate_test.go backend/internal/service/admin_service_group_test.go backend/internal/service/api_key_auth_cache_openai_fast_test.go backend/internal/service/free_ultrafast_customer_cost_test.go; do + if ! git -C "$tree" cat-file -e "$candidate:$target" 2>/dev/null; then + continue + fi + candidate_blob=$(git -C "$tree" rev-parse "$candidate:$target") + test "$(git -C "$tree" hash-object "$target")" = "$candidate_blob" + if git -C "$tree" cat-file -e "$baseline:$target" 2>/dev/null; then + expected=$(git -C "$tree" rev-parse "$baseline:$target") + git -C "$tree" show "$baseline:$target" > "$tree/$target" + test "$(git -C "$tree" hash-object "$target")" = "$expected" + printf 'historical group fixture path=%s candidate_blob=%s baseline_blob=%s contract_blob=%s\n' "$target" "$candidate_blob" "$expected" "$expected_contract" + else + exclude_absent_fixture "$target" "$contract" + fi + done +fi + # Older Account/search contracts cannot compile the unrelated new search # fixture in a service-package RED. Preserve the exact historical Account leaf; # only exclude the test when its baseline absence and candidate blob are proven. diff --git a/.github/workflows/ws-followup-capture.yml b/.github/workflows/ws-followup-capture.yml new file mode 100644 index 000000000000..139ffad2354a --- /dev/null +++ b/.github/workflows/ws-followup-capture.yml @@ -0,0 +1,78 @@ +name: WS followup error capture +on: + pull_request: + paths: + - 'backend/internal/service/openai_ws_forwarder_ingress.go' + - 'backend/internal/service/openai_ws_v2_passthrough_adapter.go' + - 'backend/internal/service/openai_ws_request_capture.go' + - 'backend/internal/requestcapture/websocket.go' + - 'backend/internal/handler/openai_ws_followup_capture_test.go' + - 'backend/internal/requestcapture/followup_test_support.go' + - 'backend/internal/service/openai_ws_agent_recovery.go' + - 'backend/internal/service/openai_ws_history_diagnostics_test.go' + - '.github/workflows/ws-followup-capture.yml' +permissions: + contents: read +jobs: + capture: + runs-on: ubuntu-24.04 + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + phase: [RED, GREEN] + steps: + - uses: actions/checkout@v6 + with: + ref: ${{ github.event.pull_request.head.sha }} + fetch-depth: 0 + persist-credentials: false + - uses: actions/setup-go@v6 + with: + go-version-file: backend/go.mod + cache-dependency-path: backend/go.sum + - name: Restore original readers + if: matrix.phase == 'RED' + run: | + for file in backend/internal/service/openai_ws_forwarder_ingress.go backend/internal/service/openai_ws_v2_passthrough_adapter.go backend/internal/service/openai_ws_request_capture.go backend/internal/requestcapture/websocket.go; do + git show "2927209f5492d8cb690b83a1964b2f093c19ad7a:$file" > "$file" + done + - name: Successful first turn followed by captured error + env: + PHASE: ${{ matrix.phase }} + shell: bash + run: | + set -euo pipefail + set +e + (cd backend && go test -race -json -tags=unit,plaintextowner,followupcapture ./internal/handler -run '^TestOpenAIWSFollowupErrorCapture$' -count=1 -timeout=3m) > "$RUNNER_TEMP/ws-followup-capture.jsonl" 2>&1 + result=$? + set -e + cat "$RUNNER_TEMP/ws-followup-capture.jsonl" + python3 .github/scripts/assert-readable-continuation.py --clean-log "$RUNNER_TEMP/ws-followup-capture.jsonl" + python3 - "$PHASE" "$result" "$RUNNER_TEMP/ws-followup-capture.jsonl" <<'PY' + import json,sys + phase,code,path=sys.argv[1],int(sys.argv[2]),sys.argv[3] + rows=[json.loads(l) for l in open(path) if l.startswith('{')] + names={'TestOpenAIWSFollowupErrorCapture/'+m for m in ('http_bridge','ctx_pool','shared','dedicated','passthrough')} + if phase=='RED': + assert code!=0 + for name in names: + assert any(r.get('Action')=='fail' and r.get('Test')==name for r in rows),name + # Old passthrough captures the upstream failure into turn 1; + # the other modes discard turn 2 after the first terminal. + marker = 'FOLLOWUP_ERROR_CAPTURE_WRONG_TURN' if name.endswith('/passthrough') else 'FOLLOWUP_ERROR_CAPTURE_MISSING' + assert marker in ''.join(r.get('Output','') for r in rows if r.get('Test')==name),name + else: + assert code==0 + for name in names: assert any(r.get('Action')=='pass' and r.get('Test')==name for r in rows),name + PY + - name: Restore refusal diagnostics contain no task or response identities + if: matrix.phase == 'GREEN' + working-directory: backend + run: go test -race -tags=unit,followupcapture ./internal/service -run '^TestOpenAIWSUnsentHistoryDiagnostics$' -count=1 -timeout=2m + - uses: actions/upload-artifact@v7 + if: always() + with: + name: ws-followup-capture-${{ matrix.phase }}-${{ github.event.pull_request.head.sha }} + path: ${{ runner.temp }}/ws-followup-capture.jsonl + if-no-files-found: error diff --git a/backend/internal/handler/openai_ws_followup_capture_test.go b/backend/internal/handler/openai_ws_followup_capture_test.go new file mode 100644 index 000000000000..9c9c789964ba --- /dev/null +++ b/backend/internal/handler/openai_ws_followup_capture_test.go @@ -0,0 +1,136 @@ +//go:build unit && plaintextowner && followupcapture + +package handler + +import ( + "context" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/Wei-Shaw/sub2api/internal/config" + "github.com/Wei-Shaw/sub2api/internal/requestcapture" + "github.com/Wei-Shaw/sub2api/internal/server/middleware" + "github.com/Wei-Shaw/sub2api/internal/service" + coderws "github.com/coder/websocket" + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/require" + "github.com/tidwall/gjson" +) + +func TestOpenAIWSFollowupErrorCapture(t *testing.T) { + for _, mode := range readableWSModes { + t.Run(mode, func(t *testing.T) { + var calls atomic.Int64 + response := func() []byte { + if calls.Add(1) == 1 { + return []byte(`{"type":"response.completed","response":{"id":"resp_capture_first","status":"completed","output":[{"type":"message","role":"assistant","content":[{"type":"output_text","text":"first success"}]}],"usage":{"input_tokens":2,"output_tokens":1}}}`) + } + return []byte(`{"type":"response.failed","response":{"id":"resp_capture_second","status":"failed","error":{"code":"invalid_request_error","message":"fixture second turn rejected"}}}`) + } + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if !strings.EqualFold(r.Header.Get("Upgrade"), "websocket") { + _, _ = io.ReadAll(r.Body) + w.Header().Set("Content-Type", "text/event-stream") + fmt.Fprintf(w, "data: %s\n\n", response()) + return + } + conn, e := coderws.Accept(w, r, nil) + if e != nil { + t.Error(e) + return + } + defer conn.CloseNow() + for { + _, _, e := conn.Read(r.Context()) + if e != nil { + return + } + if conn.Write(r.Context(), coderws.MessageText, response()) != nil { + return + } + } + })) + defer upstream.Close() + h := newOpenAIWSPassthroughHandlerHarnessWithKey(t, upstream.URL, http.Header{}, map[string]string{service.SettingKeyCyberSessionBlockEnabled: "false"}, nil, func(key *service.APIKey) { + // This fixture tests an upstream follow-up rejection. Opaque + // response pointers require permission even with tiny local input. + key.AllowLongContext = true + }, func(a *service.Account, cfg *config.Config, _ *service.HTTPUpstream, _ *service.AccountRepository) { + a.Extra["openai_apikey_responses_websockets_v2_mode"] = mode + cfg.Gateway.OpenAIWS.IngressInterTurnIdleTimeoutSeconds = 15 + }) + issue600Finish(t, h, http.Header{}) // close constructor's idle socket; reuse real handler dependencies + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + store := &recoveryCaptureStore{tasks: map[string][]byte{}, records: map[string][]byte{}} + directory := t.TempDir() + manager, e := requestcapture.New(store, directory, requestcapture.Config{Enabled: true, QuotaMiB: 8, RetentionDays: 1}) + require.NoError(t, e) + defer manager.Close() + task, e := manager.Create(ctx, requestcapture.CreateTask{TargetType: "user", TargetID: h.apiKey.UserID, DurationMinutes: 1}, "fixture") + require.NoError(t, e) + observedSession := make(chan *requestcapture.Session, 1) + router := gin.New() + router.Use(func(c *gin.Context) { + c.Set(string(middleware.ContextKeyAPIKey), h.apiKey) + c.Set(string(middleware.ContextKeyUser), middleware.AuthSubject{UserID: h.apiKey.UserID, Concurrency: 1}) + c.Next() + }) + router.Use(middleware.RequestCapture(manager)) + router.Use(func(c *gin.Context) { + observedSession <- requestcapture.FromContext(c.Request.Context()) + if requestcapture.FromContext(c.Request.Context()) == nil { + t.Error("fixture capture session was not admitted") + } + c.Next() + }) + router.GET("/v1/responses", h.handler.ResponsesWebSocket) + server := httptest.NewServer(router) + defer server.Close() + conn, _, e := coderws.Dial(ctx, "ws"+strings.TrimPrefix(server.URL, "http")+"/v1/responses", &coderws.DialOptions{HTTPHeader: http.Header{"Session_id": {"fixture-capture"}, "Thread-Id": {"fixture-capture-thread"}}}) + require.NoError(t, e) + defer conn.CloseNow() + session := <-observedSession + issue715WSSend(t, conn, `{"type":"response.create","model":"gpt-5.1","store":true,"input":[{"role":"user","content":"fixture first"}]}`) + require.Equal(t, "response.completed", gjson.GetBytes(issue715WSRead(t, conn), "type").String()) + second := `{"type":"response.create","model":"gpt-5.1","store":true,"previous_response_id":"resp_capture_first","input":[{"role":"user","content":"fixture second"}]}` + issue715WSSend(t, conn, second) + terminal := issue715WSRead(t, conn) + require.Equal(t, "response.failed", gjson.GetBytes(terminal, "type").String()) + require.Equal(t, "fixture second turn rejected", gjson.GetBytes(terminal, "response.error.message").String(), "fixture must reach the fake upstream: %s", terminal) + require.Equal(t, int64(2), calls.Load()) + _ = conn.CloseNow() + require.Eventually(t, func() bool { return manager.Stats().ActiveRequests == 0 }, 3*time.Second, 5*time.Millisecond) + records, e := manager.Records(ctx, task.ID, "", true, 10, 0) + require.NoError(t, e) + t.Logf("actual capture lifecycle: %+v", requestcapture.FollowupTestState(session)) + taskState, taskErr := manager.Task(ctx, task.ID) + require.NoError(t, taskErr) + require.Len(t, records, 1, "FOLLOWUP_ERROR_CAPTURE_MISSING: only second failed turn must be retained; stats=%+v task=%+v", manager.Stats(), taskState) + record := records[0] + require.Equal(t, 2, record.Turn, "FOLLOWUP_ERROR_CAPTURE_WRONG_TURN: the second response must not be attributed to the first turn") + require.True(t, record.IsError) + require.False(t, record.Partial, record.Reason) + parts := map[string][]byte{} + for _, part := range record.Parts { + body, e := os.ReadFile(filepath.Join(directory, task.ID, record.ID, part.Name)) + require.NoError(t, e) + parts[part.Stage] = body + } + require.Len(t, parts, 4) + require.JSONEq(t, second, string(parts["client_request"])) + require.Contains(t, string(parts["upstream_request"]), "fixture second") + require.Contains(t, string(parts["client_response"]), "fixture second turn rejected") + require.NotContains(t, string(parts["client_response"]), "first success") + require.Equal(t, int64(2), calls.Load()) + }) + } +} diff --git a/backend/internal/requestcapture/followup_test_support.go b/backend/internal/requestcapture/followup_test_support.go new file mode 100644 index 000000000000..dac49e962d4f --- /dev/null +++ b/backend/internal/requestcapture/followup_test_support.go @@ -0,0 +1,10 @@ +//go:build followupcapture + +package requestcapture + +// FollowupTestState is compiled only into the isolated capture regression. +func FollowupTestState(s *Session) map[string]any { + s.mu.Lock() + defer s.mu.Unlock() + return map[string]any{"turn": s.turn, "turn_ended": s.turnEnded, "error": s.resultError, "failed": s.failed, "closed": s.closed, "reason": s.reason, "outcome": s.clientOutcome, "protocol": s.meta.Protocol, "pending": s.pending.Load()} +} diff --git a/backend/internal/requestcapture/websocket.go b/backend/internal/requestcapture/websocket.go index 827f520b8aa2..e9ca3dbf7cd1 100644 --- a/backend/internal/requestcapture/websocket.go +++ b/backend/internal/requestcapture/websocket.go @@ -67,12 +67,34 @@ func (s *Session) Frame(stage string, attempt int, payload []byte) { return } s.frameMu.Lock() + defer s.frameMu.Unlock() + s.frameLocked(stage, attempt, payload) +} + +// Commit a successfully written client frame before the next inbound turn can +// reset its capture state. The actual write retains the caller's timeout. +func (s *Session) DeliverClientFrame(payload []byte, write func() error) error { + if s == nil { + return write() + } + s.frameMu.Lock() + defer s.frameMu.Unlock() + err := write() + if err != nil { + s.MarkPartial("client_write_failed") + s.MarkError("client_write_failed") + return err + } + s.frameLocked("client_response", 0, payload) + return nil +} + +func (s *Session) frameLocked(stage string, attempt int, payload []byte) { s.mu.Lock() turn := s.turn closed := s.closed || s.failed || s.turnEnded s.mu.Unlock() if closed { - s.frameMu.Unlock() return } if s.frameStreams == nil { @@ -102,7 +124,6 @@ func (s *Session) Frame(stage string, attempt int, payload []byte) { if stage == "client_response" && terminal { s.endTurnLocked() } - s.frameMu.Unlock() } func isTerminalFrame(payload []byte) bool { switch gjson.GetBytes(payload, "type").String() { diff --git a/backend/internal/service/openai_ws_agent_recovery.go b/backend/internal/service/openai_ws_agent_recovery.go index fd7d062e250c..11be9c97bbec 100644 --- a/backend/internal/service/openai_ws_agent_recovery.go +++ b/backend/internal/service/openai_ws_agent_recovery.go @@ -471,7 +471,7 @@ func (r *openAIWSAgentRecovery) write(payload []byte, write func() error) error if r.localHistory.pending != nil { r.localHistory.outputBytes += len(payload) if r.localHistory.outputBytes > openAIWSLocalHistoryLimit { - r.localHistory = openAIWSLocalHistory{} + r.localHistory = openAIWSLocalHistory{diagnostic: "output_limit"} r.accountLocalHistory() } else { r.localHistory.output.AddEvent(event, payload) @@ -644,6 +644,7 @@ const openAIWSLocalHistoryLimit = 8 << 20 var openAIWSLocalHistoryBytes atomic.Int64 type openAIWSLocalHistory struct { + diagnostic string // Fixed enum only; never retains caller content. outputBytes int tools json.RawMessage committed []json.RawMessage @@ -740,15 +741,21 @@ func openAIWSHistoryCopy(items []json.RawMessage) ([]json.RawMessage, bool) { } func (r *openAIWSAgentRecovery) stageLocalHistory(source []byte) { defer r.accountLocalHistory() + previousDiagnostic := r.localHistory.diagnostic r.localHistory.pending = nil r.localHistory.outputBytes = 0 r.localHistory.output = openAIWSLocalOutput{} + r.localHistory.diagnostic = "staging" if r.service.settingService == nil || !r.service.settingService.IsOpenAI409OmitEncryptedHistoryEnabled(r.ctx) { - r.localHistory = openAIWSLocalHistory{} + r.localHistory = openAIWSLocalHistory{diagnostic: "disabled"} return } items, ok := openAIWSPlainItems(source) if !ok || gjson.GetBytes(source, "conversation").Exists() { + r.localHistory.diagnostic = "source_unsupported" + if gjson.GetBytes(source, "conversation").Exists() { + r.localHistory.diagnostic = "conversation_reference" + } r.localHistory.ready = false return } @@ -756,6 +763,11 @@ func (r *openAIWSAgentRecovery) stageLocalHistory(source []byte) { if previous != "" { anchor := r.localHistory.anchor if !r.localHistory.ready || previous != anchor.response || r.request.keyID != anchor.key || r.request.userID != anchor.user || r.request.groupID != anchor.group || r.request.session != anchor.session || r.request.model != anchor.model { + if r.localHistory.ready || previousDiagnostic == "" { + r.localHistory.diagnostic = "prior_anchor_mismatch" + } else { + r.localHistory.diagnostic = previousDiagnostic + } r.localHistory.ready = false return } @@ -773,13 +785,14 @@ func (r *openAIWSAgentRecovery) stageLocalHistory(source []byte) { r.localHistory.tools = append(json.RawMessage(nil), value.Raw...) } if len(r.localHistory.tools) > openAIWSLocalHistoryLimit { - r.localHistory = openAIWSLocalHistory{} + r.localHistory = openAIWSLocalHistory{diagnostic: "tools_limit"} return } r.localHistory.pending, ok = openAIWSHistoryCopy(items) r.localHistory.ready = false if !ok { r.localHistory.pending = nil + r.localHistory.diagnostic = "input_limit" } if model := gjson.GetBytes(source, "model").String(); model != "" { r.localHistory.clientModel = model @@ -791,6 +804,11 @@ func (r *openAIWSAgentRecovery) commitLocalHistory(completed bool) { if !completed || h.pending == nil || strings.TrimSpace(r.completedBridge.response) == "" { h.ready = false h.committed = nil + if !completed { + h.diagnostic = "not_completed" + } else if h.pending != nil { + h.diagnostic = "response_anchor_missing" + } return } output := h.output.items @@ -804,19 +822,23 @@ func (r *openAIWSAgentRecovery) commitLocalHistory(completed bool) { if e != nil { h.ready = false h.committed = nil + h.diagnostic = "serialize_failed" return } normalized, ok := openAIWSPlainItems(append(append([]byte(`{"input":`), encoded...), '}')) if !ok { h.ready = false h.committed = nil + h.diagnostic = "output_unsupported" return } output = normalized all := combineOpenAIWSReplayItems(h.pending, output) h.committed, ok = openAIWSHistoryCopy(all) h.ready = ok + h.diagnostic = "history_limit" if ok { + h.diagnostic = "ready" h.anchor = r.completedBridge } h.pending = nil @@ -833,15 +855,32 @@ func (r *openAIWSAgentRecovery) restoreUnsentHistory(err error) error { } var unsent *openAIWSUnsentTurnError var admission *OpenAITurnAdmissionError - if !errors.As(err, &unsent) || unsent.keepAccount || !errors.As(err, &admission) { + if !errors.As(err, &unsent) { return err } r.mu.Lock() defer r.mu.Unlock() h := &r.localHistory + reason := "history_unready" + restored := false + defer func() { + if !restored { + logger.FromContext(r.ctx).Warn("openai.ws_unsent_history_restore_skipped", + zap.String("restore_reason", reason), zap.String("history_state", h.diagnostic), + zap.Int("turn", unsent.turn), zap.Int("history_items", len(h.committed)), + zap.Int64("history_bytes", r.historyReserved), zap.Bool("history_ready", h.ready), + zap.Bool("delivery_pending", r.finish != nil), + zap.Bool("source_previous_present", gjson.GetBytes(unsent.body, "previous_response_id").String() != "")) + } + }() + if unsent.keepAccount || !errors.As(err, &admission) { + reason = "not_admission" + return err + } if !h.ready || r.finish != nil || r.service.settingService == nil || !r.service.settingService.IsOpenAI409OmitEncryptedHistoryEnabled(r.ctx) { return err } + reason = "response_anchor" source := unsent.body previous := gjson.GetBytes(source, "previous_response_id") // Two blank IDs describe no continuation. Only the client's explicit @@ -850,13 +889,16 @@ func (r *openAIWSAgentRecovery) restoreUnsentHistory(err error) error { strings.TrimSpace(h.anchor.response) == "" || previous.String() != h.anchor.response || gjson.GetBytes(source, "conversation").Exists() { return err } + reason = "client_model" if unsent.model != "" && unsent.model != h.clientModel { return err } + reason = "principal_or_owner" scope := openAIAgentRecoveryFromContext(r.service.WithOpenAIAgentMessageRecovery(r.ctx, r.c, source, r.session, h.anchor.model)) if scope.userID <= 0 || scope.keyID <= 0 || scope.userID != h.anchor.user || scope.keyID != h.anchor.key || scope.groupID != h.anchor.group || scope.session != h.anchor.session || scope.lookupErr != nil || scope.conflict || scope.disallowLearning { return err } + reason = "current_input_unsupported" delta, ok := openAIWSPlainItems(source) if !ok { return err @@ -865,19 +907,23 @@ func (r *openAIWSAgentRecovery) restoreUnsentHistory(err error) error { if !openAIWSRawItemsHasPrefix(delta, h.committed) { all = combineOpenAIWSReplayItems(h.committed, delta) } + reason = "history_limit" all, ok = openAIWSHistoryCopy(all) if !ok { return err } + reason = "serialization" expanded, e := setOpenAIWSPayloadInputSequence(source, all, true) if e != nil { return err } + reason = "tool_pairing" if _, why := inspectOpenAIWSCompletedTools(expanded); why != "" { return err } // Every executed historical call must be one actually delivered on this // manager with a successful terminal verdict, including encrypted anchors. + reason = "tool_provenance" for _, item := range gjson.GetBytes(expanded, "input").Array() { kind := item.Get("type").String() if kind != "function_call" && kind != "custom_tool_call" { @@ -889,17 +935,20 @@ func (r *openAIWSAgentRecovery) restoreUnsentHistory(err error) error { return err } } + reason = "tool_declarations" if !gjson.GetBytes(expanded, "tools").Exists() && len(h.tools) > 0 { expanded, e = sjson.SetRawBytes(expanded, "tools", h.tools) if e != nil { return err } } + reason = "readable_intent" if !openAI409HasReadableIntent(expanded) { return err } // Completed calls become inert history, so a replacement provider cannot // execute the original tool call again. Keep arguments/results as exact JSON. + reason = "projection" expanded, e = projectOpenAIWSCompletedTools(expanded) if e != nil { return err @@ -908,6 +957,7 @@ func (r *openAIWSAgentRecovery) restoreUnsentHistory(err error) error { if e != nil || len(expanded) > openAIWSLocalHistoryLimit { return err } + restored = true copy := *unsent copy.body = expanded h.ready = false // proof consumed once; no reuse after any downstream failure @@ -929,7 +979,7 @@ func (r *openAIWSAgentRecovery) accountLocalHistory() { for { old := openAIWSLocalHistoryBytes.Load() if old+delta > 128<<20 { - r.localHistory = openAIWSLocalHistory{} + r.localHistory = openAIWSLocalHistory{diagnostic: "budget_limit"} openAIWSLocalHistoryBytes.Add(-r.historyReserved) r.historyReserved = 0 return diff --git a/backend/internal/service/openai_ws_forwarder_ingress.go b/backend/internal/service/openai_ws_forwarder_ingress.go index 32c94fe8e8ba..0cf4da22ab3e 100644 --- a/backend/internal/service/openai_ws_forwarder_ingress.go +++ b/backend/internal/service/openai_ws_forwarder_ingress.go @@ -15,6 +15,7 @@ import ( "github.com/Wei-Shaw/sub2api/internal/pkg/openai" "github.com/Wei-Shaw/sub2api/internal/pkg/requesttiming" + "github.com/Wei-Shaw/sub2api/internal/requestcapture" coderws "github.com/coder/websocket" "github.com/gin-gonic/gin" "github.com/tidwall/gjson" @@ -468,6 +469,9 @@ func (r *openAIWSIngressReader) read(ctx context.Context) (coderws.MessageType, // Frames preserved during a key wait keep their original order. if frame, ok := r.popWaitFrame(); ok { r.lastReadAt.Store(&frame.receivedAt) + capture := requestcapture.FromContext(ctx) + capture.SetProtocol("websocket") + capture.ClientFrame(frame.payload) return frame.messageType, frame.payload, nil } select { @@ -482,6 +486,9 @@ func (r *openAIWSIngressReader) read(ctx context.Context) (coderws.MessageType, return 0, nil, r.err case frame := <-r.frames: r.lastReadAt.Store(&frame.receivedAt) + capture := requestcapture.FromContext(ctx) + capture.SetProtocol("websocket") + capture.ClientFrame(frame.payload) return frame.messageType, frame.payload, nil } } @@ -1137,6 +1144,7 @@ func (s *OpenAIGatewayService) ProxyResponsesWebSocketFromClient( eventBytes := buildOpenAIFastPolicyBlockedWSEvent(blocked) if eventBytes != nil { writeCtx, cancel := newOpenAIWSDownstreamWriteContext(ctx, hooks, s.openAIWSWriteTimeout()) + writeCtx = requestcapture.WithSession(writeCtx, requestcapture.FromContext(c.Request.Context())) _ = WriteCapturedWSClient(writeCtx, clientConn, coderws.MessageText, eventBytes) cancel() } @@ -1183,6 +1191,7 @@ func (s *OpenAIGatewayService) ProxyResponsesWebSocketFromClient( writeClientMessage := func(message []byte) error { writeCtx, cancel := newOpenAIWSDownstreamWriteContext(ctx, hooks, s.openAIWSWriteTimeout()) + writeCtx = requestcapture.WithSession(writeCtx, requestcapture.FromContext(c.Request.Context())) defer cancel() message = restoreCodexToolNamesFromContext(c, message) return agentRecovery.write(message, func() error { diff --git a/backend/internal/service/openai_ws_history_diagnostics_test.go b/backend/internal/service/openai_ws_history_diagnostics_test.go new file mode 100644 index 000000000000..0abf11d0478f --- /dev/null +++ b/backend/internal/service/openai_ws_history_diagnostics_test.go @@ -0,0 +1,52 @@ +//go:build unit && followupcapture + +package service + +import ( + "context" + "encoding/json" + "testing" + + "github.com/Wei-Shaw/sub2api/internal/config" + "github.com/Wei-Shaw/sub2api/internal/pkg/logger" + "github.com/stretchr/testify/require" + "go.uber.org/zap" + "go.uber.org/zap/zaptest/observer" +) + +func TestOpenAIWSUnsentHistoryDiagnostics(t *testing.T) { + for _, tc := range []struct{ name, input, state, reason string }{ + {"unreadable_history", `[{"type":"agent_message","encrypted_content":"fixture-private-cipher"}]`, "source_unsupported", "history_unready"}, + {"wrong_response", `[{"role":"user","content":"fixture-private-task"}]`, "ready", "response_anchor"}, + } { + t.Run(tc.name, func(t *testing.T) { + core, entries := observer.New(zap.WarnLevel) + ctx := logger.IntoContext(context.Background(), zap.New(core)) + svc := &OpenAIGatewayService{cfg: &config.Config{}, settingService: NewSettingService(&codexPolicyMigrationRepoStub{values: map[string]string{SettingKeyOpenAI409OmitEncryptedHistoryEnabled: "true"}}, nil)} + manager, c, _ := issue715WSManager(t, svc, ctx) + defer manager.abort() + SetOpenAIWSRecoveryPrincipal(c, 715, 715) + manager.request = openAIAgentRecoveryFromContext(manager.ctx) + manager.stageLocalHistory([]byte(`{"model":"gpt-6-astra","input":` + tc.input + `}`)) + manager.completedBridge.response = "resp_fixture_private" + manager.commitLocalHistory(true) + current := []byte(`{"type":"response.create","model":"gpt-6-astra","previous_response_id":"resp_fixture_other","input":[{"type":"function_call_output","call_id":"fixture-private-call","output":"fixture-private-result"}]}`) + cause := openAIWSUnsentTurn(denyOpenAITurn("account_ineligible"), current, "gpt-6-astra", 7) + require.Same(t, cause, manager.restoreUnsentHistory(cause)) + logs := entries.FilterMessage("openai.ws_unsent_history_restore_skipped").All() + require.Len(t, logs, 1) + fields := logs[0].ContextMap() + require.Equal(t, tc.state, fields["history_state"]) + require.Equal(t, tc.reason, fields["restore_reason"]) + require.Equal(t, int64(7), fields["turn"]) + for name := range fields { + require.Contains(t, []string{"restore_reason", "history_state", "turn", "history_items", "history_bytes", "history_ready", "delivery_pending", "source_previous_present"}, name) + } + raw, err := json.Marshal(fields) + require.NoError(t, err) + for _, secret := range []string{"fixture-private", "resp_fixture", "encrypted_content", "call_id"} { + require.NotContains(t, string(raw), secret) + } + }) + } +} diff --git a/backend/internal/service/openai_ws_request_capture.go b/backend/internal/service/openai_ws_request_capture.go index 234e8daf9837..1d84d1f22592 100644 --- a/backend/internal/service/openai_ws_request_capture.go +++ b/backend/internal/service/openai_ws_request_capture.go @@ -42,15 +42,8 @@ func (c *captureUpstreamFrameConn) ReadFrame(ctx context.Context) (coderws.Messa // WriteCapturedWSClient observes successfully delivered downstream frames. func WriteCapturedWSClient(ctx context.Context, conn *coderws.Conn, t coderws.MessageType, payload []byte) error { - err := conn.Write(ctx, t, payload) capture := requestcapture.FromContext(ctx) - if err == nil { - capture.Frame("client_response", 0, payload) - } else { - capture.MarkPartial("client_write_failed") - capture.MarkError("client_write_failed") - } - return err + return capture.DeliverClientFrame(payload, func() error { return conn.Write(ctx, t, payload) }) } func captureWSLeaseWrite(ctx context.Context, account int64, headers http.Header, value any, write func() error) error { capture := requestcapture.FromContext(ctx) diff --git a/backend/internal/service/openai_ws_v2_passthrough_adapter.go b/backend/internal/service/openai_ws_v2_passthrough_adapter.go index 59b7368c6a14..5bf76bd64066 100644 --- a/backend/internal/service/openai_ws_v2_passthrough_adapter.go +++ b/backend/internal/service/openai_ws_v2_passthrough_adapter.go @@ -619,6 +619,9 @@ func (c *openAIWSClientFrameConn) ReadFrame(ctx context.Context) (coderws.Messag // original order ahead of anything still in the channel. if frame, ok := c.reader.popWaitFrame(); ok { c.reader.lastReadAt.Store(&frame.receivedAt) + capture := requestcapture.FromContext(ctx) + capture.SetProtocol("websocket") + capture.ClientFrame(frame.payload) return frame.messageType, frame.payload, nil } if timeout == nil && c.waitingForNextTurn.Load() && c.interTurnIdleTimeout > 0 { @@ -637,6 +640,9 @@ func (c *openAIWSClientFrameConn) ReadFrame(ctx context.Context) (coderws.Messag return 0, nil, c.reader.err case frame := <-c.reader.frames: c.reader.lastReadAt.Store(&frame.receivedAt) + capture := requestcapture.FromContext(ctx) + capture.SetProtocol("websocket") + capture.ClientFrame(frame.payload) return frame.messageType, frame.payload, nil case <-c.interTurnStarted: case <-timeout: @@ -698,6 +704,9 @@ func (c *openAIWSClientFrameConn) WriteFrame(ctx context.Context, msgType coderw payload = c.restoreToolNames(payload) } } + if c.controlCtx != nil { + ctx = requestcapture.WithSession(ctx, requestcapture.FromContext(c.controlCtx)) + } return c.agentRecovery.write(payload, func() error { return WriteCapturedWSClient(ctx, c.conn, msgType, payload) })