Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .github/scripts/prepare-ws-owner-red-fixtures.sh
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ fi

for target in \
backend/internal/service/openai_409_omit_history.go \
backend/internal/service/openai_lossy_history_restart.go \
backend/internal/service/openai_tool_provenance.go \
backend/internal/repository/gateway_cache_tool_provenance.go \
backend/internal/service/openai_agent_message_delivery.go \
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/issue715-agent-recovery.yml
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,7 @@ jobs:
original=6df0cd5bb0ce4d774d34004008f9dd661d236468
bash .github/scripts/historical-recovery-timezone-callers.sh
targets=(backend/internal/handler/openai_gateway_handler.go backend/internal/service/openai_account_scheduler.go backend/internal/service/openai_gateway_forward.go backend/internal/service/openai_gateway_response_handling.go backend/internal/service/openai_gateway_service.go backend/internal/service/openai_ws_forwarder_ingress.go backend/internal/service/openai_ws_v2_passthrough_adapter.go backend/internal/service/openai_ws_http_bridge.go backend/internal/service/openai_ws_pool.go)
added=(backend/internal/service/openai_agent_message_recovery.go backend/internal/service/openai_agent_message_recovery_cache.go backend/internal/service/openai_agent_message_delivery.go backend/internal/repository/gateway_cache_agent_recovery.go backend/internal/repository/gateway_cache_agent_shadow_epoch.go backend/internal/service/openai_ws_agent_recovery.go backend/internal/service/openai_agent_message_recovery_wait.go backend/internal/service/openai_ws_search_activation.go backend/internal/service/openai_409_omit_history.go backend/internal/service/openai_409_readable_shadow.go backend/internal/repository/gateway_cache_readable_shadow.go backend/internal/service/openai_400_omit_history.go backend/internal/service/openai_history_omission.go)
added=(backend/internal/service/openai_agent_message_recovery.go backend/internal/service/openai_agent_message_recovery_cache.go backend/internal/service/openai_agent_message_delivery.go backend/internal/repository/gateway_cache_agent_recovery.go backend/internal/repository/gateway_cache_agent_shadow_epoch.go backend/internal/service/openai_ws_agent_recovery.go backend/internal/service/openai_agent_message_recovery_wait.go backend/internal/service/openai_ws_search_activation.go backend/internal/service/openai_409_omit_history.go backend/internal/service/openai_409_readable_shadow.go backend/internal/repository/gateway_cache_readable_shadow.go backend/internal/service/openai_400_omit_history.go backend/internal/service/openai_history_omission.go backend/internal/service/openai_lossy_history_restart.go)
# The exact original gateway struct predates analysis admission state.
added+=(backend/internal/service/openai_ws_readable_continuation.go backend/internal/service/openai_agent_replay_metadata.go backend/internal/service/openai_agent_recovery_diagnostics.go backend/internal/service/openai_tool_provenance.go backend/internal/repository/gateway_cache_tool_provenance.go)
handler_compat=aa8e596e6d6f3e4648d6beba73b39b240dd5357a
Expand Down Expand Up @@ -287,7 +287,7 @@ jobs:
scheduler_leaf=backend/internal/service/openai_account_scheduler.go
wait_added=backend/internal/service/openai_agent_message_recovery_wait.go
# The Beta helper depends on fields absent in this historical recovery struct.
omission_added=(backend/internal/service/openai_409_omit_history.go backend/internal/service/openai_409_readable_shadow.go backend/internal/repository/gateway_cache_readable_shadow.go backend/internal/repository/gateway_cache_agent_shadow_epoch.go backend/internal/service/openai_400_omit_history.go backend/internal/service/openai_history_omission.go)
omission_added=(backend/internal/service/openai_409_omit_history.go backend/internal/service/openai_409_readable_shadow.go backend/internal/repository/gateway_cache_readable_shadow.go backend/internal/repository/gateway_cache_agent_shadow_epoch.go backend/internal/service/openai_400_omit_history.go backend/internal/service/openai_history_omission.go backend/internal/service/openai_lossy_history_restart.go)
omission_added+=(backend/internal/service/openai_agent_replay_metadata.go)
projection_types=backend/internal/service/openai_ws_plaintext_projection.go
metadata_compat=aa8e596e6d6f3e4648d6beba73b39b240dd5357a
Expand Down
95 changes: 95 additions & 0 deletions .github/workflows/ws-history-shapes.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,95 @@
name: WS readable history shapes
on:
pull_request:
paths:
- 'backend/internal/service/openai_ws_agent_recovery.go'
- 'backend/internal/service/openai_lossy_history_restart*.go'
- 'backend/internal/service/openai_ws_readable_history_items*.go'
- 'backend/internal/service/openai_ws_client_history_recovery_test.go'
- 'backend/internal/service/openai_ws_readable_continuation.go'
- 'backend/internal/service/openai_409_omit_history.go'
- 'backend/internal/config/config.go'
- 'backend/internal/handler/openai_ws_history_shapes_test.go'
- 'backend/internal/handler/openai_ws_unsent_delta_test.go'
- '.github/workflows/ws-history-shapes.yml'
permissions:
contents: read
jobs:
history:
runs-on: ubuntu-24.04
timeout-minutes: 20
strategy:
fail-fast: false
matrix:
phase: [RED, GREEN]
steps:
- uses: actions/checkout@v6
with:
ref: ${{ github.event.pull_request.head.sha }}
fetch-depth: 0
persist-credentials: false
- uses: actions/setup-go@v6
with:
go-version-file: backend/go.mod
cache-dependency-path: backend/go.sum
- name: Pinned original history builder
if: matrix.phase == 'RED'
run: git show 58628d98b9e0781c1b8e3df6259ab487a1490f70:backend/internal/service/openai_ws_agent_recovery.go > backend/internal/service/openai_ws_agent_recovery.go
- name: Delivered tool delta and readable historical inputs
env:
PHASE: ${{ matrix.phase }}
run: |
set -euo pipefail
set +e
(cd backend && go test -race -json -tags=unit,historyshapes ./internal/service -run '^TestOpenAIWSHistory(ReadableShapes|UnsafeShapesRemainRejected)$' -count=1 -timeout=3m) > "$RUNNER_TEMP/history-shapes.jsonl" 2>&1
result=$?
set -e
cat "$RUNNER_TEMP/history-shapes.jsonl"
python3 .github/scripts/assert-readable-continuation.py --clean-log "$RUNNER_TEMP/history-shapes.jsonl"
python3 - "$PHASE" "$result" "$RUNNER_TEMP/history-shapes.jsonl" <<'PY'
import json,sys
phase,code,path=sys.argv[1],int(sys.argv[2]),sys.argv[3]
rows=[json.loads(l) for l in open(path) if l.startswith('{')]
assert any(r.get('Action')=='pass' and r.get('Test')=='TestOpenAIWSHistoryUnsafeShapesRemainRejected' for r in rows)
names={'TestOpenAIWSHistoryReadableShapes/'+n for n in ('readable_agent','summary_agent','named_task','compaction','compaction_summary','item_reference')}
for name in names:
assert any(r.get('Action')==('fail' if phase=='RED' else 'pass') and r.get('Test')==name for r in rows),name
if phase=='RED':assert 'READABLE_SHAPE_NOT_RESTORED' in ''.join(r.get('Output','') for r in rows if r.get('Test')==name),name
assert (code!=0) if phase=='RED' else (code==0)
PY
- name: Explicit client-readable history policy preserves data without tool grants
if: matrix.phase == 'GREEN'
working-directory: backend
run: go test -race -tags=unit,historyshapes ./internal/service -run '^TestOpenAI(WSClientHistoryRestart|LossyHistoryRestart)$' -count=1 -timeout=2m
- uses: actions/upload-artifact@v7
if: always()
with:
name: ws-history-shapes-${{ matrix.phase }}-${{ github.event.pull_request.head.sha }}
path: ${{ runner.temp }}/history-shapes.jsonl
socket:
runs-on: ubuntu-24.04
timeout-minutes: 25
steps:
- uses: actions/checkout@v6
with:
ref: ${{ github.event.pull_request.head.sha }}
persist-credentials: false
- uses: actions/setup-go@v6
with:
go-version-file: backend/go.mod
cache-dependency-path: backend/go.sum
- name: Isolated Redis dependency
run: |
if ! command -v redis-server >/dev/null; then
sudo apt-get update
sudo apt-get install -y --no-install-recommends redis-server
fi
- name: Seventh turn account pause and next-turn continuation
run: |
set -euo pipefail
(cd backend && go test -race -json -tags=unit,unsentdelta,historyshapes ./internal/handler -run '^TestOpenAIWS(ReadableHistoryShapeTakeover|ClientHistoryTakeover|LossyHistoryTakeover)$' -count=1 -timeout=5m) | tee "$RUNNER_TEMP/history-shapes-socket.jsonl"
- uses: actions/upload-artifact@v7
if: always()
with:
name: ws-history-shapes-socket-${{ github.event.pull_request.head.sha }}
path: ${{ runner.temp }}/history-shapes-socket.jsonl
16 changes: 16 additions & 0 deletions backend/internal/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -1563,6 +1563,12 @@ type GatewayOpenAIWSConfig struct {
ClientReadLimitBytes int64 `mapstructure:"client_read_limit_bytes"`
// HTTPBridgeEnabled: 首包过大时,保持客户端 WS,改用 HTTP Responses 上游。
HTTPBridgeEnabled bool `mapstructure:"http_bridge_enabled"`
// Explicit operator opt-in: unreadable provider state may be detached while
// caller-supplied completed tool history is carried as inert, unverified data.
ClientHistoryRecoveryEnabled bool `mapstructure:"client_history_recovery_enabled"`
// Explicit destructive compatibility mode for HTTP and WS: discard opaque
// provider state and carry tool history only as caller-supplied text.
LossyHistoryRestartEnabled bool `mapstructure:"lossy_history_restart_enabled"`
// HTTPBridgeThresholdBytes: 触发 HTTP bridge 的入站 WS payload 阈值。
HTTPBridgeThresholdBytes int64 `mapstructure:"http_bridge_threshold_bytes"`
// SSEAccelerationMaxPayloadBytes bounds the encoded HTTP-to-WS request, including its JSON newline.
Expand Down Expand Up @@ -2805,6 +2811,8 @@ func setDefaults() {
viper.SetDefault("gateway.openai_ws.smart_protocol.large_output_threshold", 2000)
viper.SetDefault("gateway.openai_ws.smart_protocol.long_context_bytes", 200000)
viper.SetDefault("gateway.openai_ws.http_bridge_enabled", true)
viper.SetDefault("gateway.openai_ws.client_history_recovery_enabled", false)
viper.SetDefault("gateway.openai_ws.lossy_history_restart_enabled", false)
viper.SetDefault("gateway.openai_ws.http_bridge_threshold_bytes", 15*1024*1024)
viper.SetDefault("gateway.openai_ws.sse_acceleration_max_payload_bytes", 15*1024*1024)
viper.SetDefault("gateway.openai_ws.responses_websockets", false)
Expand Down Expand Up @@ -4357,3 +4365,11 @@ func warnIfInsecureURL(field, raw string) {
slog.Warn("url uses http scheme; use https in production to avoid token leakage", "field", field)
}
}

func (c *Config) OpenAIWSClientHistoryRecoveryEnabled() bool {
return c != nil && c.Gateway.OpenAIWS.ClientHistoryRecoveryEnabled
}

func (c *Config) OpenAILossyHistoryRestartEnabled() bool {
return c != nil && c.Gateway.OpenAIWS.LossyHistoryRestartEnabled
}
33 changes: 33 additions & 0 deletions backend/internal/handler/openai_ws_history_shapes_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
//go:build unit && unsentdelta && historyshapes

package handler

import "testing"

func TestOpenAIWSReadableHistoryShapeTakeover(t *testing.T) {
for _, mode := range readableWSModes {
for _, tc := range []struct{ name, input string }{
{"named_task", `,{"type":"function_call_output","name":"task","output":"fixture delegated work"}`},
{"compaction", `,{"type":"compaction","summary":"fixture readable compacted progress","encrypted_content":"fixture-opaque-compaction"}`},
{"readable_agent", `,{"type":"agent_message","content":[{"type":"input_text","text":"fixture readable delegated progress"}],"encrypted_content":"fixture-opaque-delegation"}`},
} {
t.Run(mode+"/"+tc.name, func(t *testing.T) { runUnsentDeltaWithInput(t, mode, 7, "", tc.input) })
}
}
}

func TestOpenAIWSClientHistoryTakeover(t *testing.T) {
for _, mode := range readableWSModes {
t.Run(mode, func(t *testing.T) {
runUnsentDeltaWithPolicy(t, mode, 7, "", `,{"type":"function_call","call_id":"fixture-old-client-call","name":"fixture_old_effect","arguments":"{}","status":"completed"},{"type":"function_call_output","call_id":"fixture-old-client-call","output":"client reports effect already applied"}`, true)
})
}
}

func TestOpenAIWSLossyHistoryTakeover(t *testing.T) {
for _, mode := range readableWSModes {
t.Run(mode, func(t *testing.T) {
runUnsentDeltaWithLossyPolicy(t, mode, 7, "", `,{"type":"agent_message","encrypted_content":"fixture-unreadable-task"},{"type":"function_call_output","call_id":"orphan","output":"fixture old result"}`, false, true)
})
}
}
35 changes: 31 additions & 4 deletions backend/internal/handler/openai_ws_unsent_delta_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import (
"io"
"net/http"
"net/http/httptest"
"reflect"
"strings"
"sync"
"testing"
Expand All @@ -36,6 +37,15 @@ func TestOpenAIWSUnsentDeltaFences(t *testing.T) {
}
}
func runUnsentDelta(t *testing.T, mode string, switchTurn int, fence string) {
runUnsentDeltaWithInput(t, mode, switchTurn, fence, "")
}
func runUnsentDeltaWithInput(t *testing.T, mode string, switchTurn int, fence, extraInput string) {
runUnsentDeltaWithPolicy(t, mode, switchTurn, fence, extraInput, false)
}
func runUnsentDeltaWithPolicy(t *testing.T, mode string, switchTurn int, fence, extraInput string, clientHistory bool) {
runUnsentDeltaWithLossyPolicy(t, mode, switchTurn, fence, extraInput, clientHistory, false)
}
func runUnsentDeltaWithLossyPolicy(t *testing.T, mode string, switchTurn int, fence, extraInput string, clientHistory, lossy bool) {
t.Helper()
var mu sync.Mutex
var frames [][]byte
Expand Down Expand Up @@ -103,6 +113,17 @@ func runUnsentDelta(t *testing.T, mode string, switchTurn int, fence string) {
a.Credentials["api_key"] = "fixture-A"
a.Extra["openai_apikey_responses_websockets_v2_mode"] = mode
a.Extra["codex_fingerprint_mode"] = "off"
if clientHistory {
setClientHistory := reflect.ValueOf(&cfg.Gateway.OpenAIWS).Elem().FieldByName("ClientHistoryRecoveryEnabled")
require.True(t, setClientHistory.IsValid())
setClientHistory.SetBool(true)
}
// Optional interface keeps historical test overlays buildable.
if lossy {
setLossy := reflect.ValueOf(&cfg.Gateway.OpenAIWS).Elem().FieldByName("LossyHistoryRestartEnabled")
require.True(t, setLossy.IsValid())
setLossy.SetBool(true)
}
cfg.Gateway.MaxAccountSwitches = 2
cfg.Gateway.Scheduling.OpenAITurnAdmissionCacheTTLSeconds = 0
cfg.Gateway.OpenAIWS.IngressInterTurnIdleTimeoutSeconds = 20
Expand Down Expand Up @@ -135,7 +156,7 @@ func runUnsentDelta(t *testing.T, mode string, switchTurn int, fence string) {
raw, _ := json.Marshal(b)
return string(raw)
}
issue715WSSend(t, h.clientConn, create(`[{"role":"user","content":"Perform fixture exactly once and retain the task"},{"type":"reasoning","encrypted_content":"fixture-history-state","summary":[{"type":"summary_text","text":"fixture readable progress"}]}]`, ""))
issue715WSSend(t, h.clientConn, create(`[{"role":"user","content":"Perform fixture exactly once and retain the task"},{"type":"reasoning","encrypted_content":"fixture-history-state","summary":[{"type":"summary_text","text":"fixture readable progress"}]}`+extraInput+`]`, ""))
executions := 0
consume := func(raw []byte) {
items := gjson.GetBytes(raw, "response.output").Array()
Expand Down Expand Up @@ -231,11 +252,17 @@ func runUnsentDelta(t *testing.T, mode string, switchTurn int, fence string) {
require.Equal(t, "Bearer fixture-B", selected[switchTurn-1])
resumed := sent[switchTurn-1]
require.NotContains(t, string(resumed), "resp_delta_", "old provider pointer must not reach B")
require.Contains(t, string(resumed), "Perform fixture exactly once")
if !lossy {
require.Contains(t, string(resumed), "Perform fixture exactly once")
}
require.Contains(t, string(resumed), "fixture applied exactly once")
require.Contains(t, string(resumed), "fixture readable progress")
if !lossy {
require.Contains(t, string(resumed), "fixture readable progress")
}
require.NotContains(t, string(resumed), "fixture-history-state")
require.Contains(t, string(resumed), "9007199254740993")
if !lossy {
require.Contains(t, string(resumed), "9007199254740993")
}
require.True(t, gjson.GetBytes(resumed, "tools").IsArray(), "effective tool declarations must survive takeover")
for _, item := range gjson.GetBytes(resumed, "input").Array() {
require.NotContains(t, item.Get("type").String(), "call", "completed tools must be inert history")
Expand Down
31 changes: 30 additions & 1 deletion backend/internal/service/openai_409_omit_history.go
Original file line number Diff line number Diff line change
Expand Up @@ -129,7 +129,25 @@ func (s *OpenAIGatewayService) TryRebuildOpenAI409History(ctx context.Context, c
// Provider state and mixed completed producers cannot be forwarded as one new
// account's conversation. Rebuild before selection, after all item fences were
// read. A healthy single producer without pointers retains ordinary routing.
func (s *OpenAIGatewayService) PrepareOpenAI409ReadableContinuation(ctx context.Context, c *gin.Context, body []byte, session, model string) (context.Context, []byte, bool, error) {
func (s *OpenAIGatewayService) PrepareOpenAI409ReadableContinuation(ctx context.Context, c *gin.Context, body []byte, session, model string) (nextCtx context.Context, nextBody []byte, changed bool, resultErr error) {
if next, clean, changed := s.PrepareOpenAILossyHistoryRestart(ctx, c, body, session, model); changed {
return next, clean, true, nil
}
defer func() {
if _, ws := ctx.Value(openAIWSAgentRecoveryRetryKey{}).(*atomic.Bool); ws && resultErr != nil {
// Some protocol counterexamples restore the pre-policy WS file.
// Keep that historical implementation buildable without injecting
// the candidate recovery algorithm into its RED baseline.
if restarter, available := any(s).(interface {
restartOpenAIWSClientReadableHistory(context.Context, *gin.Context, []byte, string, string, error) (context.Context, []byte, bool)
}); available {
if next, clean, ok := restarter.restartOpenAIWSClientReadableHistory(ctx, c, body, session, model, resultErr); ok {
nextCtx, nextBody, changed, resultErr = next, clean, true, nil
}
}
}
}()

r := openAIAgentRecoveryFromContext(ctx)
if r == nil {
return ctx, body, false, nil
Expand All @@ -151,6 +169,17 @@ func (s *OpenAIGatewayService) PrepareOpenAI409ReadableContinuation(ctx context.
// It may only be discarded, under the explicit Beta policy, in favor of the
// caller's own readable task. Existing foreign ownership/store failures deny.
func (s *OpenAIGatewayService) CanDiscardOpenAI409ResponsePointer(ctx context.Context, body []byte, groupID int64, responseID string) bool {
if s != nil && s.settingService != nil {
policy, ok := any(s.cfg).(interface{ OpenAILossyHistoryRestartEnabled() bool })
if ok && policy.OpenAILossyHistoryRestartEnabled() && utf8.Valid(body) && openAIEncryptedJSONUnambiguous(body) &&
s.settingService.IsOpenAI409OmitEncryptedHistoryEnabled(ctx) {
// This grants only permission to discard the pointer. The handler
// requires a successful rebuild and verifies its removal from both
// body copies before dispatch; no foreign response is consumed.
_, changed := discardOpenAIProviderHistory(body)
return changed
}
}
if s == nil || s.settingService == nil || !openAI409HasReadableIntent(body) || !s.settingService.IsOpenAI409OmitEncryptedHistoryEnabled(ctx) {
return false
}
Expand Down
Loading