Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 24 additions & 0 deletions .github/scripts/prepare-replay-red-fixtures.sh
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,30 @@ if ! test -f "$tree/$seam"; then
exclude_absent_fixture backend/internal/handler/openai_issue842_oauth_repair_test.go backend/internal/service/openai_ws_agent_recovery.go
fi

# PR878 added a Group field used by otherwise unrelated service-package tests.
# A full old-source overlay must compile them against that same historical API;
# selected recovery oracles and all candidate GREEN tests remain untouched.
contract=backend/internal/service/group.go
if ! git -C "$tree" grep -q 'FreeOpenAIUltrafast' "$baseline" -- "$contract"; then
expected_contract=$(git -C "$tree" rev-parse "$baseline:$contract")
test "$(git -C "$tree" hash-object "$contract")" = "$expected_contract"
for target in backend/internal/service/admin_group_duplicate_test.go backend/internal/service/admin_service_group_test.go backend/internal/service/api_key_auth_cache_openai_fast_test.go backend/internal/service/free_ultrafast_customer_cost_test.go; do
if ! git -C "$tree" cat-file -e "$candidate:$target" 2>/dev/null; then
continue
fi
candidate_blob=$(git -C "$tree" rev-parse "$candidate:$target")
test "$(git -C "$tree" hash-object "$target")" = "$candidate_blob"
if git -C "$tree" cat-file -e "$baseline:$target" 2>/dev/null; then
expected=$(git -C "$tree" rev-parse "$baseline:$target")
git -C "$tree" show "$baseline:$target" > "$tree/$target"
test "$(git -C "$tree" hash-object "$target")" = "$expected"
printf 'historical group fixture path=%s candidate_blob=%s baseline_blob=%s contract_blob=%s\n' "$target" "$candidate_blob" "$expected" "$expected_contract"
else
exclude_absent_fixture "$target" "$contract"
fi
done
fi

# Older Account/search contracts cannot compile the unrelated new search
# fixture in a service-package RED. Preserve the exact historical Account leaf;
# only exclude the test when its baseline absence and candidate blob are proven.
Expand Down
78 changes: 78 additions & 0 deletions .github/workflows/ws-followup-capture.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
name: WS followup error capture
on:
pull_request:
paths:
- 'backend/internal/service/openai_ws_forwarder_ingress.go'
- 'backend/internal/service/openai_ws_v2_passthrough_adapter.go'
- 'backend/internal/service/openai_ws_request_capture.go'
- 'backend/internal/requestcapture/websocket.go'
- 'backend/internal/handler/openai_ws_followup_capture_test.go'
- 'backend/internal/requestcapture/followup_test_support.go'
- 'backend/internal/service/openai_ws_agent_recovery.go'
- 'backend/internal/service/openai_ws_history_diagnostics_test.go'
- '.github/workflows/ws-followup-capture.yml'
permissions:
contents: read
jobs:
capture:
runs-on: ubuntu-24.04
timeout-minutes: 20
strategy:
fail-fast: false
matrix:
phase: [RED, GREEN]
steps:
- uses: actions/checkout@v6
with:
ref: ${{ github.event.pull_request.head.sha }}
fetch-depth: 0
persist-credentials: false
- uses: actions/setup-go@v6
with:
go-version-file: backend/go.mod
cache-dependency-path: backend/go.sum
- name: Restore original readers
if: matrix.phase == 'RED'
run: |
for file in backend/internal/service/openai_ws_forwarder_ingress.go backend/internal/service/openai_ws_v2_passthrough_adapter.go backend/internal/service/openai_ws_request_capture.go backend/internal/requestcapture/websocket.go; do
git show "2927209f5492d8cb690b83a1964b2f093c19ad7a:$file" > "$file"
done
- name: Successful first turn followed by captured error
env:
PHASE: ${{ matrix.phase }}
shell: bash
run: |
set -euo pipefail
set +e
(cd backend && go test -race -json -tags=unit,plaintextowner,followupcapture ./internal/handler -run '^TestOpenAIWSFollowupErrorCapture$' -count=1 -timeout=3m) > "$RUNNER_TEMP/ws-followup-capture.jsonl" 2>&1
result=$?
set -e
cat "$RUNNER_TEMP/ws-followup-capture.jsonl"
python3 .github/scripts/assert-readable-continuation.py --clean-log "$RUNNER_TEMP/ws-followup-capture.jsonl"
python3 - "$PHASE" "$result" "$RUNNER_TEMP/ws-followup-capture.jsonl" <<'PY'
import json,sys
phase,code,path=sys.argv[1],int(sys.argv[2]),sys.argv[3]
rows=[json.loads(l) for l in open(path) if l.startswith('{')]
names={'TestOpenAIWSFollowupErrorCapture/'+m for m in ('http_bridge','ctx_pool','shared','dedicated','passthrough')}
if phase=='RED':
assert code!=0
for name in names:
assert any(r.get('Action')=='fail' and r.get('Test')==name for r in rows),name
# Old passthrough captures the upstream failure into turn 1;
# the other modes discard turn 2 after the first terminal.
marker = 'FOLLOWUP_ERROR_CAPTURE_WRONG_TURN' if name.endswith('/passthrough') else 'FOLLOWUP_ERROR_CAPTURE_MISSING'
assert marker in ''.join(r.get('Output','') for r in rows if r.get('Test')==name),name
else:
assert code==0
for name in names: assert any(r.get('Action')=='pass' and r.get('Test')==name for r in rows),name
PY
- name: Restore refusal diagnostics contain no task or response identities
if: matrix.phase == 'GREEN'
working-directory: backend
run: go test -race -tags=unit,followupcapture ./internal/service -run '^TestOpenAIWSUnsentHistoryDiagnostics$' -count=1 -timeout=2m
- uses: actions/upload-artifact@v7
if: always()
with:
name: ws-followup-capture-${{ matrix.phase }}-${{ github.event.pull_request.head.sha }}
path: ${{ runner.temp }}/ws-followup-capture.jsonl
if-no-files-found: error
136 changes: 136 additions & 0 deletions backend/internal/handler/openai_ws_followup_capture_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,136 @@
//go:build unit && plaintextowner && followupcapture

package handler

import (
"context"
"fmt"
"io"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"sync/atomic"
"testing"
"time"

"github.com/Wei-Shaw/sub2api/internal/config"
"github.com/Wei-Shaw/sub2api/internal/requestcapture"
"github.com/Wei-Shaw/sub2api/internal/server/middleware"
"github.com/Wei-Shaw/sub2api/internal/service"
coderws "github.com/coder/websocket"
"github.com/gin-gonic/gin"
"github.com/stretchr/testify/require"
"github.com/tidwall/gjson"
)

func TestOpenAIWSFollowupErrorCapture(t *testing.T) {
for _, mode := range readableWSModes {
t.Run(mode, func(t *testing.T) {
var calls atomic.Int64
response := func() []byte {
if calls.Add(1) == 1 {
return []byte(`{"type":"response.completed","response":{"id":"resp_capture_first","status":"completed","output":[{"type":"message","role":"assistant","content":[{"type":"output_text","text":"first success"}]}],"usage":{"input_tokens":2,"output_tokens":1}}}`)
}
return []byte(`{"type":"response.failed","response":{"id":"resp_capture_second","status":"failed","error":{"code":"invalid_request_error","message":"fixture second turn rejected"}}}`)
}
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if !strings.EqualFold(r.Header.Get("Upgrade"), "websocket") {
_, _ = io.ReadAll(r.Body)
w.Header().Set("Content-Type", "text/event-stream")
fmt.Fprintf(w, "data: %s\n\n", response())
return
}
conn, e := coderws.Accept(w, r, nil)
if e != nil {
t.Error(e)
return
}
defer conn.CloseNow()
for {
_, _, e := conn.Read(r.Context())
if e != nil {
return
}
if conn.Write(r.Context(), coderws.MessageText, response()) != nil {
return
}
}
}))
defer upstream.Close()
h := newOpenAIWSPassthroughHandlerHarnessWithKey(t, upstream.URL, http.Header{}, map[string]string{service.SettingKeyCyberSessionBlockEnabled: "false"}, nil, func(key *service.APIKey) {
// This fixture tests an upstream follow-up rejection. Opaque
// response pointers require permission even with tiny local input.
key.AllowLongContext = true
}, func(a *service.Account, cfg *config.Config, _ *service.HTTPUpstream, _ *service.AccountRepository) {
a.Extra["openai_apikey_responses_websockets_v2_mode"] = mode
cfg.Gateway.OpenAIWS.IngressInterTurnIdleTimeoutSeconds = 15
})
issue600Finish(t, h, http.Header{}) // close constructor's idle socket; reuse real handler dependencies
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
store := &recoveryCaptureStore{tasks: map[string][]byte{}, records: map[string][]byte{}}
directory := t.TempDir()
manager, e := requestcapture.New(store, directory, requestcapture.Config{Enabled: true, QuotaMiB: 8, RetentionDays: 1})
require.NoError(t, e)
defer manager.Close()
task, e := manager.Create(ctx, requestcapture.CreateTask{TargetType: "user", TargetID: h.apiKey.UserID, DurationMinutes: 1}, "fixture")
require.NoError(t, e)
observedSession := make(chan *requestcapture.Session, 1)
router := gin.New()
router.Use(func(c *gin.Context) {
c.Set(string(middleware.ContextKeyAPIKey), h.apiKey)
c.Set(string(middleware.ContextKeyUser), middleware.AuthSubject{UserID: h.apiKey.UserID, Concurrency: 1})
c.Next()
})
router.Use(middleware.RequestCapture(manager))
router.Use(func(c *gin.Context) {
observedSession <- requestcapture.FromContext(c.Request.Context())
if requestcapture.FromContext(c.Request.Context()) == nil {
t.Error("fixture capture session was not admitted")
}
c.Next()
})
router.GET("/v1/responses", h.handler.ResponsesWebSocket)
server := httptest.NewServer(router)
defer server.Close()
conn, _, e := coderws.Dial(ctx, "ws"+strings.TrimPrefix(server.URL, "http")+"/v1/responses", &coderws.DialOptions{HTTPHeader: http.Header{"Session_id": {"fixture-capture"}, "Thread-Id": {"fixture-capture-thread"}}})
require.NoError(t, e)
defer conn.CloseNow()
session := <-observedSession
issue715WSSend(t, conn, `{"type":"response.create","model":"gpt-5.1","store":true,"input":[{"role":"user","content":"fixture first"}]}`)
require.Equal(t, "response.completed", gjson.GetBytes(issue715WSRead(t, conn), "type").String())
second := `{"type":"response.create","model":"gpt-5.1","store":true,"previous_response_id":"resp_capture_first","input":[{"role":"user","content":"fixture second"}]}`
issue715WSSend(t, conn, second)
terminal := issue715WSRead(t, conn)
require.Equal(t, "response.failed", gjson.GetBytes(terminal, "type").String())
require.Equal(t, "fixture second turn rejected", gjson.GetBytes(terminal, "response.error.message").String(), "fixture must reach the fake upstream: %s", terminal)
require.Equal(t, int64(2), calls.Load())
_ = conn.CloseNow()
require.Eventually(t, func() bool { return manager.Stats().ActiveRequests == 0 }, 3*time.Second, 5*time.Millisecond)
records, e := manager.Records(ctx, task.ID, "", true, 10, 0)
require.NoError(t, e)
t.Logf("actual capture lifecycle: %+v", requestcapture.FollowupTestState(session))
taskState, taskErr := manager.Task(ctx, task.ID)
require.NoError(t, taskErr)
require.Len(t, records, 1, "FOLLOWUP_ERROR_CAPTURE_MISSING: only second failed turn must be retained; stats=%+v task=%+v", manager.Stats(), taskState)
record := records[0]
require.Equal(t, 2, record.Turn, "FOLLOWUP_ERROR_CAPTURE_WRONG_TURN: the second response must not be attributed to the first turn")
require.True(t, record.IsError)
require.False(t, record.Partial, record.Reason)
parts := map[string][]byte{}
for _, part := range record.Parts {
body, e := os.ReadFile(filepath.Join(directory, task.ID, record.ID, part.Name))
require.NoError(t, e)
parts[part.Stage] = body
}
require.Len(t, parts, 4)
require.JSONEq(t, second, string(parts["client_request"]))
require.Contains(t, string(parts["upstream_request"]), "fixture second")
require.Contains(t, string(parts["client_response"]), "fixture second turn rejected")
require.NotContains(t, string(parts["client_response"]), "first success")
require.Equal(t, int64(2), calls.Load())
})
}
}
10 changes: 10 additions & 0 deletions backend/internal/requestcapture/followup_test_support.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
//go:build followupcapture

package requestcapture

// FollowupTestState is compiled only into the isolated capture regression.
func FollowupTestState(s *Session) map[string]any {
s.mu.Lock()
defer s.mu.Unlock()
return map[string]any{"turn": s.turn, "turn_ended": s.turnEnded, "error": s.resultError, "failed": s.failed, "closed": s.closed, "reason": s.reason, "outcome": s.clientOutcome, "protocol": s.meta.Protocol, "pending": s.pending.Load()}
}
25 changes: 23 additions & 2 deletions backend/internal/requestcapture/websocket.go
Original file line number Diff line number Diff line change
Expand Up @@ -67,12 +67,34 @@ func (s *Session) Frame(stage string, attempt int, payload []byte) {
return
}
s.frameMu.Lock()
defer s.frameMu.Unlock()
s.frameLocked(stage, attempt, payload)
}

// Commit a successfully written client frame before the next inbound turn can
// reset its capture state. The actual write retains the caller's timeout.
func (s *Session) DeliverClientFrame(payload []byte, write func() error) error {
if s == nil {
return write()
}
s.frameMu.Lock()
defer s.frameMu.Unlock()
err := write()
if err != nil {
s.MarkPartial("client_write_failed")
s.MarkError("client_write_failed")
return err
}
s.frameLocked("client_response", 0, payload)
return nil
}

func (s *Session) frameLocked(stage string, attempt int, payload []byte) {
s.mu.Lock()
turn := s.turn
closed := s.closed || s.failed || s.turnEnded
s.mu.Unlock()
if closed {
s.frameMu.Unlock()
return
}
if s.frameStreams == nil {
Expand Down Expand Up @@ -102,7 +124,6 @@ func (s *Session) Frame(stage string, attempt int, payload []byte) {
if stage == "client_response" && terminal {
s.endTurnLocked()
}
s.frameMu.Unlock()
}
func isTerminalFrame(payload []byte) bool {
switch gjson.GetBytes(payload, "type").String() {
Expand Down
Loading
Loading