From 088f064ec1d30e3c95f956bbe213400626508ecf Mon Sep 17 00:00:00 2001 From: ranxi2001 Date: Mon, 16 Nov 2026 20:36:27 +0800 Subject: [PATCH 1/4] test: add inherited WS model tool-proof regression checkpoint --- .../scripts/prepare-ws-inherited-model-red.py | 73 +++ .../test_ws_inherited_model_evidence.py | 109 ++++ .../scripts/ws-inherited-model-evidence.py | 102 ++++ .github/workflows/ws-inherited-model.yml | 86 ++++ .../handler/openai_ws_inherited_model_test.go | 467 ++++++++++++++++++ 5 files changed, 837 insertions(+) create mode 100644 .github/scripts/prepare-ws-inherited-model-red.py create mode 100644 .github/scripts/test_ws_inherited_model_evidence.py create mode 100644 .github/scripts/ws-inherited-model-evidence.py create mode 100644 .github/workflows/ws-inherited-model.yml create mode 100644 backend/internal/handler/openai_ws_inherited_model_test.go diff --git a/.github/scripts/prepare-ws-inherited-model-red.py b/.github/scripts/prepare-ws-inherited-model-red.py new file mode 100644 index 000000000000..07ab89260388 --- /dev/null +++ b/.github/scripts/prepare-ws-inherited-model-red.py @@ -0,0 +1,73 @@ +"""Materialize the entire pinned baseline plus one identical candidate test. + +No historical overlay helper or existing worktree is modified. All archived +backend blobs (including dependencies and historical fixtures) are verified. +""" + +import json +import shutil +import subprocess +import sys +from pathlib import Path + +BASELINE = "da526bfbab47efd40549cab98f1069f9b2b45b9f" +FIXTURE = "backend/internal/handler/openai_ws_inherited_model_test.go" +GUARDS = "backend/internal/service/openai_ws_owner_conflict_test.go" + + +def git(*args, **kwargs): + return subprocess.check_output(["git", *args], **kwargs) + + +def prepare(destination, evidence): + destination, evidence = Path(destination).resolve(), Path(evidence).resolve() + candidate = git("rev-parse", "HEAD").decode().strip() + # A fresh directory avoids deleting or changing another writer's files. + destination.mkdir(parents=True, exist_ok=False) + evidence.mkdir(parents=True, exist_ok=True) + archive = subprocess.Popen(["git", "archive", BASELINE], stdout=subprocess.PIPE) + try: + subprocess.run(["tar", "-x", "-C", str(destination)], stdin=archive.stdout, check=True) + finally: + archive.stdout.close() + if archive.wait() != 0: + raise RuntimeError("baseline archive failed") + entries = [] + for row in git("ls-tree", "-rz", BASELINE, "--", "backend").split(b"\0"): + if not row: + continue + metadata, path = row.decode().split("\t", 1) + mode, kind, blob = metadata.split() + if kind != "blob" or mode not in {"100644", "100755"}: + raise ValueError(f"unhandled baseline entry: {path}") + entries.append((path, blob)) + paths = "".join(str(destination / path) + "\n" for path, _ in entries) + actual = git("hash-object", "--stdin-paths", input=paths.encode()).decode().splitlines() + if actual != [blob for _, blob in entries]: + raise ValueError("baseline backend blob mismatch") + if (destination / FIXTURE).exists(): + raise ValueError("fixture unexpectedly exists in pinned baseline") + fixture_blob = git("rev-parse", f"{candidate}:{FIXTURE}").decode().strip() + if git("hash-object", FIXTURE).decode().strip() != fixture_blob: + raise ValueError("dirty candidate fixture") + if Path(GUARDS).read_bytes() != (destination / GUARDS).read_bytes(): + raise ValueError("guard fixture changed; identical baseline/candidate guard tests required") + shutil.copyfile(FIXTURE, destination / FIXTURE) + if git("hash-object", str(destination / FIXTURE)).decode().strip() != fixture_blob: + raise ValueError("candidate test copy mismatch") + manifest = { + "baseline": BASELINE, "candidate": candidate, + "baseline_tree": git("rev-parse", f"{BASELINE}^{{tree}}").decode().strip(), + "baseline_backend_tree": git("rev-parse", f"{BASELINE}:backend").decode().strip(), + "candidate_fixture_blob": fixture_blob, "fixture": FIXTURE, + "guard_fixture_blob": git("hash-object", GUARDS).decode().strip(), + "verified_baseline_backend_blobs": len(entries), + "production_changes_in_red": [], "candidate_test_overlays": [FIXTURE], + } + (evidence / "source-manifest.json").write_text(json.dumps(manifest, indent=2) + "\n") + (evidence / "baseline-backend-blobs.txt").write_text("".join(f"{blob} {path}\n" for path, blob in entries)) + print(json.dumps(manifest, sort_keys=True)) + + +if __name__ == "__main__": + prepare(*sys.argv[1:]) diff --git a/.github/scripts/test_ws_inherited_model_evidence.py b/.github/scripts/test_ws_inherited_model_evidence.py new file mode 100644 index 000000000000..fa0bcba7abb5 --- /dev/null +++ b/.github/scripts/test_ws_inherited_model_evidence.py @@ -0,0 +1,109 @@ +"""Synthetic parser checks only; these are never Go RED/GREEN evidence.""" + +import importlib.util +import json +import unittest +from pathlib import Path + +SPEC = importlib.util.spec_from_file_location( + "evidence", Path(__file__).with_name("ws-inherited-model-evidence.py") +) +GATE = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(GATE) + + +def transcript(suite="handler", phase="RED"): + package, tests, red, positives, fences = GATE.contract(suite) + package = "github.com/Wei-Shaw/sub2api/internal/" + package + failures = set() + if suite == "handler" and phase == "RED": + for leaf in red: + parts = leaf.split("/") + failures.update("/".join(parts[:i]) for i in range(1, len(parts) + 1)) + failures.add(None) + rows = [{"Package": package, "Action": "start"}] + for name in sorted(tests): + rows.append({"Package": package, "Action": "run", "Test": name}) + if name in red and name in failures: + output = GATE.MARKER + " T1/T2 completed on A; T2 omitted model; O1/O2 and C delivered; proof=failed; T3 tool_provenance_unknown; A disabled; B sends=0; records unchanged" + elif name in positives: + output = "INHERITED_MODEL_TOOL_LOOP_OK: B sends=1; executions=1; A disabled" + elif name in fences: + output = "INHERITED_MODEL_FENCE_OK: B sends=0; records unchanged" + else: + output = "synthetic parent event" + rows.append({"Package": package, "Action": "output", "Test": name, "Output": output}) + rows.append({"Package": package, "Action": "fail" if name in failures else "pass", "Test": name}) + rows.append({"Package": package, "Action": "fail" if failures else "pass"}) + return rows + + +def encode(rows): + return "\n".join(json.dumps(row) for row in rows) + "\n" + + +class EvidenceTests(unittest.TestCase): + def test_complete_contracts(self): + for suite in ("handler", "guards"): + for phase in ("RED", "GREEN"): + with self.subTest(suite=suite, phase=phase): + status = int(suite == "handler" and phase == "RED") + result = GATE.validate(suite, phase, status, encode(transcript(suite, phase))) + self.assertEqual(status * 4, result["behavioral_red_leaves"]) + + def test_runtime_build_and_skip_contamination(self): + for diagnostic in ("[build failed]", "undefined: fixture", "panic: fixture", "WARNING: DATA RACE", "test timed out after 5m", "no tests to run", "--- SKIP: fixture", "signal: killed"): + for decoded in (False, True): + with self.subTest(diagnostic=diagnostic, decoded=decoded): + rows = transcript() + if decoded: + rows[2]["Output"] += "\n" + diagnostic + raw = encode(rows) + else: + raw = encode(rows) + diagnostic + with self.assertRaises((ValueError, json.JSONDecodeError)): + GATE.validate("handler", "RED", 1, raw) + + def test_missing_repeated_unrelated_and_wrong_outcomes(self): + for mutation in ("missing_control", "duplicate_run", "duplicate_terminal", "unrelated", "package", "skip", "wrong_failure", "missing_marker", "truncated", "empty"): + with self.subTest(mutation=mutation): + rows = transcript() + if mutation == "missing_control": + rows = [r for r in rows if r.get("Test") != GATE.FENCES + "/function/pending"] + elif mutation == "duplicate_run": + rows.append(dict(rows[1])) + elif mutation == "duplicate_terminal": + rows.append(dict(rows[-1])) + elif mutation == "unrelated": + rows.append({"Package": rows[0]["Package"], "Action": "fail", "Test": "TestOther"}) + elif mutation == "package": + rows[0]["Package"] += "_other" + elif mutation == "skip": + rows[3]["Action"] = "skip" + elif mutation == "wrong_failure": + for row in rows: + if row.get("Action") == "pass" and row.get("Test") == GATE.FENCES + "/function/pending": + row["Action"] = "fail" + elif mutation == "missing_marker": + for row in rows: + row["Output"] = row.get("Output", "").replace(GATE.MARKER, "unrelated assertion") + elif mutation == "truncated": + rows = rows[:-1] + elif mutation == "empty": + rows = [] + with self.assertRaises(ValueError): + GATE.validate("handler", "RED", 1, encode(rows)) + + def test_status_phase_and_malformed_input(self): + for status in (0, 2, 137): + with self.subTest(status=status), self.assertRaises(ValueError): + GATE.validate("handler", "RED", status, encode(transcript())) + with self.assertRaises(ValueError): + GATE.validate("handler", "GREEN", 0, encode(transcript())) + for raw in ("{malformed\n", "[]\n", "null\n", "not JSON\n"): + with self.subTest(raw=raw), self.assertRaises(ValueError): + GATE.validate("handler", "RED", 1, raw) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/scripts/ws-inherited-model-evidence.py b/.github/scripts/ws-inherited-model-evidence.py new file mode 100644 index 000000000000..0d4fb8a77e0a --- /dev/null +++ b/.github/scripts/ws-inherited-model-evidence.py @@ -0,0 +1,102 @@ +"""Accept only the complete authenticated inherited-model behavioral matrix.""" + +import importlib.util +import json +import sys +from collections import Counter +from pathlib import Path + +SPEC = importlib.util.spec_from_file_location( + "clean", Path(__file__).with_name("assert-readable-continuation.py") +) +CLEAN = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(CLEAN) + +POSITIVE = "TestOpenAIWSInheritedModelToolProof" +FENCES = "TestOpenAIWSInheritedModelProofFences" +SCENARIOS = ("explicit", "inherited", "mapped_explicit", "mapped_inherited") +NEGATIVES = ( + "unknown", "pending", "epoch", "arguments", "terminal_excludes", + "model_change", "same_wire_model_change", +) +KINDS = ("function", "custom") +MARKER = "INHERITED_MODEL_TOOL_PROOF_NOT_RESUMED:" + + +def contract(suite): + if suite == "handler": + roots = {POSITIVE, FENCES} + parents = {f"{root}/{kind}" for root in roots for kind in KINDS} + good = {f"{POSITIVE}/{kind}/{case}" for kind in KINDS for case in SCENARIOS} + fences = {f"{FENCES}/{kind}/{case}" for kind in KINDS for case in NEGATIVES} + red = {name for name in good if name.endswith("/inherited") or name.endswith("/mapped_inherited")} + return "handler", roots | parents | good | fences, red, good, fences + if suite == "guards": + cases = { + "TestOpenAIToolProvenanceDeliveryAndScope": ( + "completed", "early_only", "write_failed", "terminal_excludes", "duplicate", + ), + "TestOpenAIWSMixedOwnerFinalSendFences": ( + "none", "pending", "failed", "conflicted", "missing", "changed", "input", "tools", "call_proof", + ), + } + tests = set(cases) | {f"{root}/{case}" for root, leaves in cases.items() for case in leaves} + return "service", tests, set(), set(), set() + raise ValueError("unknown suite") + + +def validate(suite, phase, status, transcript): + if phase not in {"RED", "GREEN"}: + raise ValueError("unknown phase") + CLEAN.check_clean(transcript) + package, tests, red_leaves, positives, fences = contract(suite) + package = "github.com/Wei-Shaw/sub2api/internal/" + package + rows = [] + for line in transcript.splitlines(): + # Go can print dependency downloads on stderr before its JSON stream. + if line.startswith("go: downloading "): + continue + row = json.loads(line) + if not isinstance(row, dict) or row.get("Package") != package: + raise ValueError("unexpected package or malformed event") + if row.get("Action") not in {"start", "run", "pause", "cont", "output", "pass", "fail"}: + raise ValueError("unexpected action (including skip/build failure)") + if row.get("Test") not in tests | {None}: + raise ValueError("unrelated test") + rows.append(row) + counts = Counter((r.get("Action"), r.get("Test")) for r in rows) + failures = set() + if suite == "handler" and phase == "RED": + for leaf in red_leaves: + parts = leaf.split("/") + failures.update("/".join(parts[:i]) for i in range(1, len(parts) + 1)) + failures.add(None) + if status != (1 if failures else 0): + raise ValueError("wrong process exit status") + if counts["start", None] != 1 or {r.get("Test") for r in rows if r.get("Action") == "run"} != tests: + raise ValueError("missing/repeated package or test run") + for name in tests | {None}: + if name is not None and counts["run", name] != 1: + raise ValueError(f"missing/repeated run: {name}") + expected = "fail" if name in failures else "pass" + if counts[expected, name] != 1 or counts["pass", name] + counts["fail", name] != 1: + raise ValueError(f"wrong/missing/repeated outcome: {name}") + for name in positives | fences: + output = "".join(r.get("Output", "") for r in rows if r.get("Test") == name) + if name in failures: + required = ( + MARKER, "T1/T2 completed on A", "T2 omitted model", "O1/O2 and C delivered", + "proof=failed", "T3 tool_provenance_unknown", "A disabled", "B sends=0", "records unchanged", + ) + elif name in fences: + required = ("INHERITED_MODEL_FENCE_OK:", "B sends=0", "records unchanged") + else: + required = ("INHERITED_MODEL_TOOL_LOOP_OK:", "B sends=1", "executions=1", "A disabled") + if any(text not in output for text in required): + raise ValueError(f"missing behavioral evidence: {name}") + return {"suite": suite, "phase": phase, "exit_status": status, "tests": len(tests), "behavioral_red_leaves": len(red_leaves) if failures else 0} + + +if __name__ == "__main__": + suite, phase, status, path = sys.argv[1:] + print(json.dumps(validate(suite, phase, int(status), Path(path).read_text()), sort_keys=True)) diff --git a/.github/workflows/ws-inherited-model.yml b/.github/workflows/ws-inherited-model.yml new file mode 100644 index 000000000000..33c4c931f455 --- /dev/null +++ b/.github/workflows/ws-inherited-model.yml @@ -0,0 +1,86 @@ +name: WS inherited model tool proof +on: + pull_request: + paths: + - 'backend/internal/service/openai*' + - 'backend/internal/handler/openai*' + - '.github/workflows/ws-inherited-model.yml' + - '.github/scripts/*ws-inherited-model*' + - '.github/scripts/test_ws_inherited_model_evidence.py' + - '.github/scripts/assert-readable-continuation.py' + workflow_dispatch: +permissions: + contents: read +jobs: + inherited-model: + name: Inherited model ${{ matrix.phase }} + runs-on: ubuntu-24.04 + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + phase: [RED, GREEN] + env: + CANDIDATE: ${{ github.event.pull_request.head.sha || github.sha }} + PHASE: ${{ matrix.phase }} + EVIDENCE: ${{ runner.temp }}/ws-inherited-model-evidence + steps: + - uses: actions/checkout@v6 + with: + ref: ${{ env.CANDIDATE }} + fetch-depth: 0 + persist-credentials: false + - uses: actions/setup-go@v6 + with: + go-version-file: backend/go.mod + cache-dependency-path: backend/go.sum + - name: Isolated Redis fixture dependency + run: | + if ! command -v redis-server >/dev/null; then + sudo apt-get update + sudo apt-get install -y --no-install-recommends redis-server + fi + - name: Exact source and identical candidate fixture + shell: bash + run: | + set -euo pipefail + test "$(git rev-parse HEAD)" = "$CANDIDATE" + git diff --exit-code + mkdir -p "$EVIDENCE" + git fetch --no-tags --depth=1 origin da526bfbab47efd40549cab98f1069f9b2b45b9f + python3 .github/scripts/prepare-ws-inherited-model-red.py "$RUNNER_TEMP/ws-inherited-model-baseline" "$EVIDENCE" + if [ "$PHASE" = RED ]; then + echo "TEST_ROOT=$RUNNER_TEMP/ws-inherited-model-baseline" >> "$GITHUB_ENV" + else + echo "TEST_ROOT=$GITHUB_WORKSPACE" >> "$GITHUB_ENV" + fi + python3 -m unittest discover -s .github/scripts -p 'test_ws_inherited_model_evidence.py' -v > "$EVIDENCE/parser-tests.log" 2>&1 + - name: Authenticated same-socket tool loop and refusal controls + shell: bash + run: | + set -euo pipefail + set +e + (cd "$TEST_ROOT/backend" && go test -race -p=2 -json -tags=unit,inheritedmodel ./internal/handler -run '^TestOpenAIWSInheritedModel(ToolProof|ProofFences)$' -count=1 -timeout=5m) > "$EVIDENCE/handler.jsonl" 2>&1 + status=$? + set -e + printf '%s\n' "$status" > "$EVIDENCE/handler.exit" + cat "$EVIDENCE/handler.jsonl" + python3 .github/scripts/ws-inherited-model-evidence.py handler "$PHASE" "$status" "$EVIDENCE/handler.jsonl" | tee "$EVIDENCE/handler-verdict.json" + - name: Delivery, model scope and final wire integrity controls + if: ${{ (success() || failure()) && env.TEST_ROOT != '' }} + shell: bash + run: | + set -euo pipefail + set +e + (cd "$TEST_ROOT/backend" && go test -race -p=2 -json -tags=unit,ownerconflict ./internal/service -run '^Test(OpenAIToolProvenanceDeliveryAndScope|OpenAIWSMixedOwnerFinalSendFences)$' -count=1 -timeout=5m) > "$EVIDENCE/guards.jsonl" 2>&1 + status=$? + set -e + printf '%s\n' "$status" > "$EVIDENCE/guards.exit" + cat "$EVIDENCE/guards.jsonl" + python3 .github/scripts/ws-inherited-model-evidence.py guards "$PHASE" "$status" "$EVIDENCE/guards.jsonl" | tee "$EVIDENCE/guards-verdict.json" + - uses: actions/upload-artifact@v7 + if: always() + with: + name: ws-inherited-model-${{ matrix.phase }}-${{ env.CANDIDATE }} + path: ${{ env.EVIDENCE }} + if-no-files-found: error diff --git a/backend/internal/handler/openai_ws_inherited_model_test.go b/backend/internal/handler/openai_ws_inherited_model_test.go new file mode 100644 index 000000000000..4fc88ae7ad57 --- /dev/null +++ b/backend/internal/handler/openai_ws_inherited_model_test.go @@ -0,0 +1,467 @@ +//go:build unit && inheritedmodel + +package handler + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "reflect" + "slices" + "strings" + "sync" + "testing" + "time" + + "github.com/Wei-Shaw/sub2api/internal/config" + "github.com/Wei-Shaw/sub2api/internal/repository" + "github.com/Wei-Shaw/sub2api/internal/service" + coderws "github.com/coder/websocket" + "github.com/redis/go-redis/v9" + "github.com/stretchr/testify/require" + "github.com/tidwall/gjson" + "github.com/tidwall/sjson" +) + +// Keep this tag opt-in: historical production overlays must not compile a new +// fixture against an older handler harness. The dedicated CI uses a complete +// pinned source tree and copies only this identical candidate regression. +func TestOpenAIWSInheritedModelToolProof(t *testing.T) { + for _, kind := range []string{"function", "custom"} { + t.Run(kind, func(t *testing.T) { + for _, scenario := range []string{"explicit", "inherited", "mapped_explicit", "mapped_inherited"} { + t.Run(scenario, func(t *testing.T) { runWSInheritedModelToolProof(t, kind, scenario) }) + } + }) + } +} + +func TestOpenAIWSInheritedModelProofFences(t *testing.T) { + for _, kind := range []string{"function", "custom"} { + t.Run(kind, func(t *testing.T) { + for _, scenario := range []string{"unknown", "pending", "epoch", "arguments", "terminal_excludes", "model_change", "same_wire_model_change"} { + t.Run(scenario, func(t *testing.T) { runWSInheritedModelToolProof(t, kind, scenario) }) + } + }) + } +} + +// UseNumber keeps both schema and completed-result comparisons lossless. +func inheritedModelJSONEqual(a, b string) bool { + decode := func(raw string) (any, error) { + var value any + d := json.NewDecoder(strings.NewReader(raw)) + d.UseNumber() + err := d.Decode(&value) + return value, err + } + x, xerr := decode(a) + y, yerr := decode(b) + return xerr == nil && yerr == nil && reflect.DeepEqual(x, y) +} + +func inheritedModelRecords(t *testing.T, client *redis.Client) map[string]string { + t.Helper() + result := make(map[string]string) + for _, pattern := range []string{"openai_agent_owner:*", "openai_agent_shadow_epoch:*", "openai_tool_provenance:v1:*", "openai_readable_shadow:*"} { + keys, err := client.Keys(context.Background(), pattern).Result() + require.NoError(t, err) + for _, key := range keys { + raw, err := client.Get(context.Background(), key).Result() + require.NoError(t, err) + result[key] = raw + } + } + return result +} + +func runWSInheritedModelToolProof(t *testing.T, kind, scenario string) { + t.Helper() + const model = "gpt-5.1" + const otherModel = "gpt-5.2" + const task = "Apply the fixture ledger entry exactly once, then report the complete receipt." + inherited := strings.HasSuffix(scenario, "inherited") + mapped := strings.HasPrefix(scenario, "mapped_") || scenario == "same_wire_model_change" + positive := inherited || scenario == "explicit" || scenario == "mapped_explicit" + wireModel := model + if mapped { + wireModel = "gpt-5.3" + } + declarations := `[{"type":"function","name":"fixture_ledger","strict":true,"parameters":{"type":"object","properties":{"n":{"type":"integer","maximum":9007199254740993},"memo":{"type":"string"}},"required":["n","memo"],"additionalProperties":false}}]` + callType, argumentField := "function_call", "arguments" + if kind == "custom" { + declarations = `[{"type":"custom","name":"fixture_ledger","description":"Apply one isolated fixture ledger entry","format":{"type":"text"}}]` + callType, argumentField = "custom_tool_call", "input" + } + arguments := `{"n":9007199254740993,"memo":"fixture 雪\nsecond line"}` + issuedCall := httpRebuildJSON(map[string]any{"type": callType, "call_id": "call_inherited_once", "name": "fixture_ledger", "status": "completed", argumentField: arguments}) + // This result is constructed only after the client receives and executes C. + var completedResult string + var mu sync.Mutex + var frames [][]byte + var identities []string + aDisabled := false + providerResponse := func(auth string, body []byte) [][]byte { + mu.Lock() + defer mu.Unlock() + frames = append(frames, append([]byte(nil), body...)) + identities = append(identities, auth) + n := len(frames) + invalid := func(reason string) [][]byte { + t.Errorf("INHERITED_PROVIDER_CONTRACT: %s", reason) + return [][]byte{[]byte(`{"type":"response.failed","response":{"status":"failed","error":{"code":"fixture_contract_rejected"}}}`)} + } + if !inheritedModelJSONEqual(declarations, gjson.GetBytes(body, "tools").Raw) { + return invalid("actual upstream must receive the complete original tool declaration/schema") + } + if gjson.GetBytes(body, "model").String() != wireModel { + return invalid("client model and mapped upstream model must remain distinct") + } + var output []json.RawMessage + var events [][]byte + switch n { + case 1: + if auth != "Bearer fixture-inherited-A" || aDisabled { + return invalid("T1 must reach enabled A") + } + output = []json.RawMessage{json.RawMessage(httpRebuildAgentA)} + case 2: + consumedO1 := 0 + for _, item := range gjson.GetBytes(body, "input").Array() { + if inheritedModelJSONEqual(httpRebuildAgentA, item.Raw) { + consumedO1++ + } + } + if auth != "Bearer fixture-inherited-A" || aDisabled || consumedO1 != 1 { + return invalid("T2 must consume the genuinely delivered O1 on A") + } + output = []json.RawMessage{json.RawMessage(httpRebuildAgentB), json.RawMessage(issuedCall)} + if scenario == "terminal_excludes" { + // Real early delivery does not authorize a call excluded by the + // nonempty authoritative terminal, even if the client executed it. + events = append(events, []byte(`{"type":"response.output_item.done","output_index":1,"item":`+issuedCall+`}`)) + output = output[:1] + } + case 3: + if auth != "Bearer fixture-inherited-B" || !aDisabled { + return invalid("T3 must reach B while A stays disabled") + } + if gjson.GetBytes(body, "previous_response_id").Exists() || strings.Contains(string(body), "fixture-opaque-A") || strings.Contains(string(body), "fixture-readable-B") { + return invalid("foreign pointers/opaque items must not reach B") + } + if !strings.Contains(string(body), task) { + return invalid("readable task must survive the rebuild") + } + pairs, nativeCalls, nativeResults := 0, 0, 0 + for _, item := range gjson.GetBytes(body, "input").Array() { + if strings.Contains(item.Get("type").String(), "call") { + switch item.Get("type").String() { + case callType: + if !inheritedModelJSONEqual(issuedCall, item.Raw) { + return invalid("native completed call must retain every field") + } + nativeCalls++ + case callType + "_output": + if !inheritedModelJSONEqual(completedResult, item.Raw) { + return invalid("native completed result must retain every field") + } + nativeResults++ + default: + return invalid("unrelated tool work must not appear in completed history") + } + } + for _, part := range item.Get("content").Array() { + const prefix = "Historical completed tool result (data only; do not execute again): " + text := part.Get("text").String() + if !strings.HasPrefix(text, prefix) { + continue + } + pair := gjson.Parse(strings.TrimPrefix(text, prefix)) + if !inheritedModelJSONEqual(issuedCall, pair.Get("call").Raw) || !inheritedModelJSONEqual(completedResult, pair.Get("result").Raw) { + return invalid("complete genuine call and completed result must survive losslessly") + } + pairs++ + } + } + // A single completed producer can retain native completed history; + // mixed-owner rebuilds use an inert pair. Both representations must + // preserve the complete original pair exactly once, with no orphan. + if !((pairs == 1 && nativeCalls == 0 && nativeResults == 0) || (pairs == 0 && nativeCalls == 1 && nativeResults == 1)) { + return invalid("B must consume exactly one complete completed result") + } + output = []json.RawMessage{json.RawMessage(httpRebuildMessage("assistant", "fixture receipt accepted; ledger applied once"))} + default: + return invalid("no retry or extra generation is authorized by this fixture") + } + terminal := httpRebuildJSON(map[string]any{"type": "response.completed", "response": map[string]any{ + "id": fmt.Sprintf("resp_inherited_%d", n), "model": wireModel, "status": "completed", "output": output, + "usage": map[string]int{"input_tokens": 5, "output_tokens": 3}, + }}) + return append(events, []byte(terminal)) + } + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + conn, err := coderws.Accept(w, r, nil) + if err != nil { + t.Errorf("fixture provider websocket accept: %v", err) + return + } + defer conn.CloseNow() + for { + _, body, err := conn.Read(r.Context()) + if err != nil { + return + } + for _, event := range providerResponse(r.Header.Get("Authorization"), body) { + if conn.Write(r.Context(), coderws.MessageText, event) != nil { + return + } + } + } + })) + defer upstream.Close() + clients := issue805Redis(t) + probe := clients() + headers := http.Header{"Authorization": {"Bearer sk-handler-cyber-test"}, "Session_id": {"fixture-inherited-session"}, "Thread-Id": {"fixture-inherited-thread"}} + var accounts *readableWSAccounts + h := newOpenAIWSPassthroughHandlerHarnessWithAuth(t, upstream.URL, headers, + map[string]string{service.SettingKeyCyberSessionBlockEnabled: "false", service.SettingKeyOpenAI409OmitEncryptedHistoryEnabled: "true"}, + repository.NewGatewayCache(clients()), func(k *service.APIKey) { k.AllowLongContext = true }, nil, + func(k *service.APIKey, cfg *config.Config) *service.APIKeyService { + k.Status = service.StatusActive + k.User.Concurrency = 1 + k.Group = &service.Group{ID: *k.GroupID, Platform: service.PlatformOpenAI, Status: service.StatusActive, Hydrated: true} + return service.NewAPIKeyService(&wsRevalidationKeyRepoStub{current: k}, nil, nil, nil, nil, nil, cfg) + }, func(a *service.Account, cfg *config.Config, _ *service.HTTPUpstream, repo *service.AccountRepository) { + a.ID, a.Priority = 1, 0 + a.Credentials["api_key"] = "fixture-inherited-A" + a.Extra["openai_apikey_responses_websockets_v2_mode"] = service.OpenAIWSIngressModeCtxPool + a.Extra["codex_fingerprint_mode"] = "off" + cfg.Gateway.MaxAccountSwitches = 2 + cfg.Gateway.Scheduling.OpenAITurnAdmissionCacheTTLSeconds = 0 + cfg.Gateway.OpenAIWS.IngressInterTurnIdleTimeoutSeconds = 20 + b := *a + b.ID, b.Priority = 2, 1 + b.Credentials = map[string]any{"api_key": "fixture-inherited-B", "base_url": upstream.URL} + if mapped { + // Two explicit client models share the same upstream model. That + // must never permit one client's model scope to borrow the other. + a.Credentials["model_mapping"] = map[string]any{model: wireModel, otherModel: wireModel} + b.Credentials["model_mapping"] = map[string]any{model: wireModel, otherModel: wireModel} + } + accounts = &readableWSAccounts{issue600SwitchRepo: &issue600SwitchRepo{&openAIWSFailoverHandlerAccountRepoStub{accounts: []service.Account{*a, b}}}} + *repo = accounts + }) + finished := make(chan struct{}) + go func() { <-h.handlerDone; close(finished) }() + defer func() { + _ = h.clientConn.CloseNow() + select { + case <-finished: + case <-time.After(5 * time.Second): + t.Error("fixture handler did not finish") + } + }() + awaitUsage := func() { + select { + case usage := <-h.usageLogs: + require.Equal(t, model, usage.RequestedModel, "billing must retain the effective client model") + case <-time.After(5 * time.Second): + t.Fatal("completed fixture turn did not settle usage") + } + } + create := func(history []string, requested string) string { + body := map[string]any{"type": "response.create", "store": true, "tools": json.RawMessage(declarations), "input": json.RawMessage("[" + strings.Join(history, ",") + "]")} + if requested != "" { + body["model"] = requested + } + return httpRebuildJSON(body) + } + executions := 0 + ledger := make(map[string]string) + var deliveredCall string + readTurn := func() ([]byte, error) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + var refusal []byte + for { + _, raw, err := h.clientConn.Read(ctx) + if err != nil { + return refusal, err + } + event := gjson.GetBytes(raw, "type").String() + items := gjson.GetBytes(raw, "response.output").Array() + if event == "response.output_item.done" { + items = append(items, gjson.GetBytes(raw, "item")) + } + for _, item := range items { + if item.Get("type").String() != "function_call" && item.Get("type").String() != "custom_tool_call" { + continue + } + require.True(t, inheritedModelJSONEqual(issuedCall, item.Raw), "client must execute the actual provider-issued call") + require.Zero(t, executions, "completed tool must never be issued/executed twice") + entry := gjson.Parse(item.Get(argumentField).String()) + require.Equal(t, "9007199254740993", entry.Get("n").Raw) + require.Equal(t, "fixture 雪\nsecond line", entry.Get("memo").String()) + require.NotContains(t, ledger, entry.Get("n").Raw) + ledger[entry.Get("n").Raw] = entry.Get("memo").String() + executions++ + deliveredCall = item.Raw + mu.Lock() + completedResult = httpRebuildJSON(map[string]any{"type": callType + "_output", "call_id": item.Get("call_id").String(), "status": "completed", "output": fmt.Sprintf("receipt=fixture-once; n=%s\n%s; complete tail retained", entry.Get("n").Raw, ledger[entry.Get("n").Raw])}) + mu.Unlock() + } + switch event { + case "error": + refusal = raw + case "response.completed", "response.failed": + return raw, nil + } + } + } + // T1 and T2 intentionally use this same authenticated client socket. + history := []string{httpRebuildMessage("user", task)} + issue715WSSend(t, h.clientConn, create(history, model)) + first, err := readTurn() + require.NoError(t, err) + require.Equal(t, "response.completed", gjson.GetBytes(first, "type").String()) + require.Equal(t, "completed", gjson.GetBytes(first, "response.status").String()) + require.JSONEq(t, httpRebuildAgentA, gjson.GetBytes(first, "response.output.0").Raw) + awaitUsage() + history = append(history, gjson.GetBytes(first, "response.output.0").Raw, httpRebuildMessage("user", "Apply the ledger entry now and keep all receipt fields.")) + t2Model := model + if inherited { + t2Model = "" + } + t2 := create(history, t2Model) + require.Equal(t, !inherited, gjson.Get(t2, "model").Exists(), "the regression must omit model on the actual client frame") + issue715WSSend(t, h.clientConn, t2) + second, err := readTurn() + require.NoError(t, err) + require.Equal(t, "response.completed", gjson.GetBytes(second, "type").String()) + require.Equal(t, "completed", gjson.GetBytes(second, "response.status").String()) + require.JSONEq(t, httpRebuildAgentB, gjson.GetBytes(second, "response.output.0").Raw) + require.Equal(t, 1, executions) + awaitUsage() + history = append(history, gjson.GetBytes(second, "response.output.0").Raw, deliveredCall, completedResult) + + // Wait for the real delivery commit, never seed a completed owner or proof. + var proofKey, proofRaw string + require.Eventually(t, func() bool { + owners := issue805Owners(t, probe) + if len(owners) != 2 { + return false + } + for _, raw := range owners { + if !issue805CompletedOwner(raw, 1) { + return false + } + } + keys, err := probe.Keys(context.Background(), "openai_tool_provenance:v1:*").Result() + require.NoError(t, err) + if len(keys) != 1 { + return false + } + proofKey = keys[0] + proofRaw, err = probe.Get(context.Background(), proofKey).Result() + require.NoError(t, err) + return proofRaw == "failed" || json.Valid([]byte(proofRaw)) + }, 5*time.Second, 10*time.Millisecond) + if !inherited && scenario != "terminal_excludes" { + require.True(t, json.Valid([]byte(proofRaw)), "explicit control must earn its real completed proof before any negative mutation") + require.NotEmpty(t, gjson.Get(proofRaw, "delivery").String()) + require.Len(t, gjson.Get(proofRaw, "anchors").Map(), 2, "proof must bind both consumed O1 and delivered O2") + } + if scenario == "terminal_excludes" { + require.Equal(t, "failed", proofRaw) + } + // Negative fixtures only revoke/change already earned evidence. No fixture + // inserts a ready proof, a completed owner or an upstream call into storage. + switch scenario { + case "unknown": + require.NoError(t, probe.Del(context.Background(), proofKey).Err()) + case "pending": + require.NoError(t, probe.Set(context.Background(), proofKey, "pending:fixture-unconfirmed", time.Minute).Err()) + case "epoch": + keys, err := probe.Keys(context.Background(), "openai_agent_shadow_epoch:*").Result() + require.NoError(t, err) + require.Len(t, keys, 2) + for _, key := range keys { + require.NoError(t, probe.Set(context.Background(), key, "ready:pending:fixture-reassociated", time.Minute).Err()) + } + case "arguments": + // Mutate only the replay of a genuinely received call, never the proof. + history[len(history)-2], err = sjson.Set(deliveredCall, argumentField, `{"n":1,"memo":"changed"}`) + require.NoError(t, err) + } + before := inheritedModelRecords(t, probe) + accounts.disable(1) + mu.Lock() + aDisabled = true + mu.Unlock() + t3Model := model + if scenario == "model_change" || scenario == "same_wire_model_change" { + t3Model = otherModel + } + issue715WSSend(t, h.clientConn, create(history, t3Model)) + terminal, readErr := readTurn() + completed := readErr == nil && gjson.GetBytes(terminal, "type").String() == "response.completed" + if completed { + awaitUsage() + } + // Await handler exit before asserting zero future B sends or unchanged + // records. A timeout/late retry is a fixture failure, never behavioral RED. + if completed { + _ = h.clientConn.CloseNow() + } + select { + case <-finished: + case <-time.After(5 * time.Second): + t.Fatal("fixture handler must settle before final evidence checks") + } + a, err := accounts.GetByID(context.Background(), 1) + require.NoError(t, err) + require.False(t, a.Schedulable, "A must stay disabled through final handler exit") + mu.Lock() + sent, routes := append([][]byte(nil), frames...), append([]string(nil), identities...) + mu.Unlock() + require.GreaterOrEqual(t, len(sent), 2) + require.Equal(t, []string{"Bearer fixture-inherited-A", "Bearer fixture-inherited-A"}, routes[:2]) + require.Equal(t, 1, executions) + require.Equal(t, map[string]string{"9007199254740993": "fixture 雪\nsecond line"}, ledger) + if !positive || !completed { + require.False(t, completed, "a fenced call must never finish on B") + require.Equal(t, []string{"Bearer fixture-inherited-A", "Bearer fixture-inherited-A"}, routes, "zero additional A/B generations after refusal") + require.Equal(t, before, inheritedModelRecords(t, probe), "refusal must preserve all owner/proof/epoch/shadow records") + require.NotEqual(t, context.DeadlineExceeded, readErr, "timeout is not a safety refusal") + require.NotEqual(t, coderws.StatusCode(-1), coderws.CloseStatus(readErr), "a real server close must terminate the refused request") + var subreasons []string + for _, entry := range h.requestLogs.FilterMessage("openai.agent_recovery_refused").All() { + if subreason, ok := entry.ContextMap()["subreason"].(string); ok { + subreasons = append(subreasons, subreason) + } + } + if positive { + require.Equal(t, "failed", proofRaw) + require.Contains(t, subreasons, "tool_provenance_unknown") + t.Fatalf("INHERITED_MODEL_TOOL_PROOF_NOT_RESUMED: T1/T2 completed on A; T2 omitted model; O1/O2 and C delivered; proof=failed; T3 tool_provenance_unknown; A disabled; B sends=0; records unchanged") + } + if scenario == "unknown" || scenario == "arguments" || scenario == "terminal_excludes" { + require.Contains(t, subreasons, "tool_provenance_unknown") + } + if scenario == "pending" { + // Expiry can be observed by the timer or by Redis on the next + // lookup. Both must fail closed with the record still pending. + require.True(t, slices.Contains(subreasons, "tool_provenance_unknown") || slices.Contains(subreasons, "tool_provenance_unavailable")) + } + t.Logf("INHERITED_MODEL_FENCE_OK: %s; A sends=2; B sends=0; records unchanged", scenario) + return + } + require.NoError(t, readErr) + require.Equal(t, "completed", gjson.GetBytes(terminal, "response.status").String()) + require.Contains(t, string(terminal), "fixture receipt accepted; ledger applied once") + require.Equal(t, []string{"Bearer fixture-inherited-A", "Bearer fixture-inherited-A", "Bearer fixture-inherited-B"}, routes) + require.Equal(t, before, inheritedModelRecords(t, probe), "B must not rewrite the original owner or delivery proof") + t.Log("INHERITED_MODEL_TOOL_LOOP_OK: A sends=2; B sends=1; complete result accepted; executions=1; A disabled") +} From 95065a5ef4810cbae96951c2190bd1ae5b3deda9 Mon Sep 17 00:00:00 2001 From: ranxi2001 Date: Mon, 16 Nov 2026 21:04:51 +0800 Subject: [PATCH 2/4] ci: initialize inherited-model evidence on runner --- .github/workflows/ws-inherited-model.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ws-inherited-model.yml b/.github/workflows/ws-inherited-model.yml index 33c4c931f455..21cb7754060a 100644 --- a/.github/workflows/ws-inherited-model.yml +++ b/.github/workflows/ws-inherited-model.yml @@ -23,8 +23,14 @@ jobs: env: CANDIDATE: ${{ github.event.pull_request.head.sha || github.sha }} PHASE: ${{ matrix.phase }} - EVIDENCE: ${{ runner.temp }}/ws-inherited-model-evidence steps: + - name: Initialize evidence directory + shell: bash + run: | + set -euo pipefail + export EVIDENCE="$RUNNER_TEMP/ws-inherited-model-evidence" + echo "EVIDENCE=$EVIDENCE" >> "$GITHUB_ENV" + mkdir -p "$EVIDENCE" - uses: actions/checkout@v6 with: ref: ${{ env.CANDIDATE }} From 2441777a761cd9097bacf43311a3e9011e15b37b Mon Sep 17 00:00:00 2001 From: ranxi2001 Date: Mon, 16 Nov 2026 21:15:55 +0800 Subject: [PATCH 3/4] fix: preserve effective client model in WS tool delivery proofs --- .../test_ws_inherited_model_evidence.py | 2 +- .../scripts/ws-inherited-model-evidence.py | 7 ++- .github/workflows/ws-inherited-model.yml | 10 +++- .../handler/openai_gateway_handler.go | 1 + .../service/openai_agent_message_recovery.go | 6 +++ .../openai_ws_recovery_client_model.go | 30 ++++++++++++ .../openai_ws_recovery_client_model_test.go | 48 +++++++++++++++++++ 7 files changed, 100 insertions(+), 4 deletions(-) create mode 100644 backend/internal/service/openai_ws_recovery_client_model.go create mode 100644 backend/internal/service/openai_ws_recovery_client_model_test.go diff --git a/.github/scripts/test_ws_inherited_model_evidence.py b/.github/scripts/test_ws_inherited_model_evidence.py index fa0bcba7abb5..12f358f2891e 100644 --- a/.github/scripts/test_ws_inherited_model_evidence.py +++ b/.github/scripts/test_ws_inherited_model_evidence.py @@ -44,7 +44,7 @@ def encode(rows): class EvidenceTests(unittest.TestCase): def test_complete_contracts(self): - for suite in ("handler", "guards"): + for suite in ("handler", "guards", "guards_with_client_model"): for phase in ("RED", "GREEN"): with self.subTest(suite=suite, phase=phase): status = int(suite == "handler" and phase == "RED") diff --git a/.github/scripts/ws-inherited-model-evidence.py b/.github/scripts/ws-inherited-model-evidence.py index 0d4fb8a77e0a..8478ccca27c2 100644 --- a/.github/scripts/ws-inherited-model-evidence.py +++ b/.github/scripts/ws-inherited-model-evidence.py @@ -31,7 +31,7 @@ def contract(suite): fences = {f"{FENCES}/{kind}/{case}" for kind in KINDS for case in NEGATIVES} red = {name for name in good if name.endswith("/inherited") or name.endswith("/mapped_inherited")} return "handler", roots | parents | good | fences, red, good, fences - if suite == "guards": + if suite in {"guards", "guards_with_client_model"}: cases = { "TestOpenAIToolProvenanceDeliveryAndScope": ( "completed", "early_only", "write_failed", "terminal_excludes", "duplicate", @@ -40,6 +40,11 @@ def contract(suite): "none", "pending", "failed", "conflicted", "missing", "changed", "input", "tools", "call_proof", ), } + if suite == "guards_with_client_model": + cases["TestOpenAIWSRecoveryEffectiveClientModel"] = ( + "no_ws_preflight", "inherited", "explicit_switch", "inherited_after_switch", + "explicit_overrides_preflight", "separate_connection", + ) tests = set(cases) | {f"{root}/{case}" for root, leaves in cases.items() for case in leaves} return "service", tests, set(), set(), set() raise ValueError("unknown suite") diff --git a/.github/workflows/ws-inherited-model.yml b/.github/workflows/ws-inherited-model.yml index 21cb7754060a..f2968987136f 100644 --- a/.github/workflows/ws-inherited-model.yml +++ b/.github/workflows/ws-inherited-model.yml @@ -77,13 +77,19 @@ jobs: shell: bash run: | set -euo pipefail + suite=guards + pattern='^Test(OpenAIToolProvenanceDeliveryAndScope|OpenAIWSMixedOwnerFinalSendFences)$' + if [ "$PHASE" = GREEN ]; then + suite=guards_with_client_model + pattern='^Test(OpenAIToolProvenanceDeliveryAndScope|OpenAIWSMixedOwnerFinalSendFences|OpenAIWSRecoveryEffectiveClientModel)$' + fi set +e - (cd "$TEST_ROOT/backend" && go test -race -p=2 -json -tags=unit,ownerconflict ./internal/service -run '^Test(OpenAIToolProvenanceDeliveryAndScope|OpenAIWSMixedOwnerFinalSendFences)$' -count=1 -timeout=5m) > "$EVIDENCE/guards.jsonl" 2>&1 + (cd "$TEST_ROOT/backend" && go test -race -p=2 -json -tags=unit,ownerconflict,inheritedmodel ./internal/service -run "$pattern" -count=1 -timeout=5m) > "$EVIDENCE/guards.jsonl" 2>&1 status=$? set -e printf '%s\n' "$status" > "$EVIDENCE/guards.exit" cat "$EVIDENCE/guards.jsonl" - python3 .github/scripts/ws-inherited-model-evidence.py guards "$PHASE" "$status" "$EVIDENCE/guards.jsonl" | tee "$EVIDENCE/guards-verdict.json" + python3 .github/scripts/ws-inherited-model-evidence.py "$suite" "$PHASE" "$status" "$EVIDENCE/guards.jsonl" | tee "$EVIDENCE/guards-verdict.json" - uses: actions/upload-artifact@v7 if: always() with: diff --git a/backend/internal/handler/openai_gateway_handler.go b/backend/internal/handler/openai_gateway_handler.go index a0ea1ad215de..fee12b073e35 100644 --- a/backend/internal/handler/openai_gateway_handler.go +++ b/backend/internal/handler/openai_gateway_handler.go @@ -3786,6 +3786,7 @@ func (h *OpenAIGatewayHandler) ResponsesWebSocket(c *gin.Context) { if err := h.concurrencyHelper.RevalidateTurnAuth(turnCtx); err != nil { return mapOpenAIWSTurnAdmissionError(err) } + service.SetOpenAIWSRecoveryClientModel(c, model) return nil }, BeforeRequestWithClientPayload: func(turn int, payload, clientPayload []byte, originalModel string) error { diff --git a/backend/internal/service/openai_agent_message_recovery.go b/backend/internal/service/openai_agent_message_recovery.go index d93d5223fdf4..7da5b063f9e8 100644 --- a/backend/internal/service/openai_agent_message_recovery.go +++ b/backend/internal/service/openai_agent_message_recovery.go @@ -116,6 +116,12 @@ func (s *OpenAIGatewayService) WithOpenAIAgentMessageRecovery(ctx context.Contex request.userID = openAIReadableRecoveryUser(c, keyID) request.httpResponsePointer = openAIAgentRecoveryHTTPUser(c, keyID) > 0 request.shadowModel = gjson.GetBytes(body, "model").String() + if strings.TrimSpace(request.shadowModel) == "" { + // A follow-up WS frame may legally inherit its client model. Use the + // authenticated current-frame preflight value, never the mapped model + // argument or the connection's initial model after an explicit switch. + request.shadowModel = openAIWSRecoveryClientModelFromContext(c) + } request.shadowReadableIntent = openAI409HasReadableIntent(body) request.shadowObservation, _ = ctx.Value(openAIReadableShadowContextKey{}).(*openAIReadableShadowObservation) if request.shadowObservation != nil { diff --git a/backend/internal/service/openai_ws_recovery_client_model.go b/backend/internal/service/openai_ws_recovery_client_model.go new file mode 100644 index 000000000000..fe7acdbb562e --- /dev/null +++ b/backend/internal/service/openai_ws_recovery_client_model.go @@ -0,0 +1,30 @@ +package service + +import ( + "strings" + + "github.com/gin-gonic/gin" +) + +const openAIWSRecoveryClientModelKey = "openai_ws_recovery_client_model" + +type openAIWSRecoveryClientModel struct{ model string } + +// SetOpenAIWSRecoveryClientModel records the current frame's effective client +// model after the handler's fresh auth preflight. The ingress resolves omitted +// models from its last accepted client model before invoking that preflight. +// Keep this separate from channel/account mapping and from diagnostic fields. +func SetOpenAIWSRecoveryClientModel(c *gin.Context, model string) { + if c != nil { + c.Set(openAIWSRecoveryClientModelKey, openAIWSRecoveryClientModel{strings.TrimSpace(model)}) + } +} + +func openAIWSRecoveryClientModelFromContext(c *gin.Context) string { + if c == nil { + return "" + } + value, _ := c.Get(openAIWSRecoveryClientModelKey) + current, _ := value.(openAIWSRecoveryClientModel) + return current.model +} diff --git a/backend/internal/service/openai_ws_recovery_client_model_test.go b/backend/internal/service/openai_ws_recovery_client_model_test.go new file mode 100644 index 000000000000..c49090c8dd8f --- /dev/null +++ b/backend/internal/service/openai_ws_recovery_client_model_test.go @@ -0,0 +1,48 @@ +//go:build unit && inheritedmodel + +package service + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" +) + +// Client scope must follow the current accepted frame across explicit model +// changes, even when both aliases map to the same upstream model. Normalizing +// proof metadata must never change immutable client input or the forward scope. +func TestOpenAIWSRecoveryEffectiveClientModel(t *testing.T) { + for _, scenario := range []string{"no_ws_preflight", "inherited", "explicit_switch", "inherited_after_switch", "explicit_overrides_preflight", "separate_connection"} { + t.Run(scenario, func(t *testing.T) { + c, _ := newTurnStateTestContext(t, 715, "fixture-session") + svc := &OpenAIGatewayService{} + body := []byte(`{"input":[{"role":"user","content":"fixture scope"}]}`) + want := "" + if scenario != "no_ws_preflight" { + SetOpenAIWSRecoveryClientModel(c, "client-first") + want = "client-first" + } + switch scenario { + case "explicit_switch", "inherited_after_switch": + SetOpenAIWSRecoveryClientModel(c, "client-second") + want = "client-second" + if scenario == "explicit_switch" { + body = []byte(`{"model":"client-second","input":[]}`) + } + case "explicit_overrides_preflight": + body = []byte(`{"model":"client-explicit","input":[]}`) + want = "client-explicit" + case "separate_connection": + c, _ = newTurnStateTestContext(t, 715, "fixture-session") + want = "" + } + before := string(body) + ctx := svc.WithOpenAIAgentMessageRecovery(context.Background(), c, body, "fixture-session", "mapped-upstream") + request := openAIAgentRecoveryFromContext(ctx) + require.Equal(t, want, request.shadowModel) + require.Equal(t, "mapped-upstream", request.model) + require.Equal(t, before, string(body), "the source frame must remain byte-identical") + }) + } +} From 30dc698901ea9a4aa6711d64df21f647d56838b3 Mon Sep 17 00:00:00 2001 From: ranxi2001 Date: Mon, 16 Nov 2026 22:06:30 +0800 Subject: [PATCH 4/4] fix: retain observed pending owner when its wait expires --- .../test_ws_inherited_model_evidence.py | 35 +++- .../scripts/ws-inherited-model-evidence.py | 18 +- .github/workflows/ws-inherited-model.yml | 78 +++++++++ .../service/openai_agent_message_delivery.go | 7 +- .../openai_agent_pending_observation_test.go | 160 ++++++++++++++++++ 5 files changed, 290 insertions(+), 8 deletions(-) create mode 100644 backend/internal/service/openai_agent_pending_observation_test.go diff --git a/.github/scripts/test_ws_inherited_model_evidence.py b/.github/scripts/test_ws_inherited_model_evidence.py index 12f358f2891e..ba6b4a61a83e 100644 --- a/.github/scripts/test_ws_inherited_model_evidence.py +++ b/.github/scripts/test_ws_inherited_model_evidence.py @@ -16,7 +16,7 @@ def transcript(suite="handler", phase="RED"): package, tests, red, positives, fences = GATE.contract(suite) package = "github.com/Wei-Shaw/sub2api/internal/" + package failures = set() - if suite == "handler" and phase == "RED": + if red and phase == "RED": for leaf in red: parts = leaf.split("/") failures.update("/".join(parts[:i]) for i in range(1, len(parts) + 1)) @@ -24,7 +24,9 @@ def transcript(suite="handler", phase="RED"): rows = [{"Package": package, "Action": "start"}] for name in sorted(tests): rows.append({"Package": package, "Action": "run", "Test": name}) - if name in red and name in failures: + if suite == "pending" and name in positives: + output = "PENDING_OBSERVATION_LOST: owner_pending=1" if name in failures else "PENDING_OBSERVATION_CONTROL_OK: classification preserved; no ownership publication" + elif name in red and name in failures: output = GATE.MARKER + " T1/T2 completed on A; T2 omitted model; O1/O2 and C delivered; proof=failed; T3 tool_provenance_unknown; A disabled; B sends=0; records unchanged" elif name in positives: output = "INHERITED_MODEL_TOOL_LOOP_OK: B sends=1; executions=1; A disabled" @@ -44,12 +46,35 @@ def encode(rows): class EvidenceTests(unittest.TestCase): def test_complete_contracts(self): - for suite in ("handler", "guards", "guards_with_client_model"): + for suite in ("handler", "guards", "guards_with_client_model", "pending"): for phase in ("RED", "GREEN"): with self.subTest(suite=suite, phase=phase): - status = int(suite == "handler" and phase == "RED") + status = int(suite in {"handler", "pending"} and phase == "RED") result = GATE.validate(suite, phase, status, encode(transcript(suite, phase))) - self.assertEqual(status * 4, result["behavioral_red_leaves"]) + self.assertEqual(status * len(GATE.contract(suite)[2]), result["behavioral_red_leaves"]) + + def test_pending_requires_exact_failure_and_all_error_controls(self): + for change in ("marker", "missing_backend_error", "failed_control", "skip", "duplicate", "wrong_status"): + with self.subTest(change=change): + rows = transcript("pending") + status = 1 + if change == "marker": + for row in rows: + row["Output"] = row.get("Output", "").replace("PENDING_OBSERVATION_LOST:", "other failure") + elif change == "missing_backend_error": + rows = [r for r in rows if r.get("Test") != GATE.PENDING_ROOT + "/pending_backend_deadline"] + elif change == "failed_control": + for row in rows: + if row.get("Test") == GATE.PENDING_ROOT + "/pending_backend_deadline" and row["Action"] == "pass": + row["Action"] = "fail" + elif change == "skip": + rows[3]["Action"] = "skip" + elif change == "duplicate": + rows.append(rows[-1]) + else: + status = 0 + with self.assertRaises(ValueError): + GATE.validate("pending", "RED", status, encode(rows)) def test_runtime_build_and_skip_contamination(self): for diagnostic in ("[build failed]", "undefined: fixture", "panic: fixture", "WARNING: DATA RACE", "test timed out after 5m", "no tests to run", "--- SKIP: fixture", "signal: killed"): diff --git a/.github/scripts/ws-inherited-model-evidence.py b/.github/scripts/ws-inherited-model-evidence.py index 8478ccca27c2..5b568bba2927 100644 --- a/.github/scripts/ws-inherited-model-evidence.py +++ b/.github/scripts/ws-inherited-model-evidence.py @@ -21,9 +21,19 @@ ) KINDS = ("function", "custom") MARKER = "INHERITED_MODEL_TOOL_PROOF_NOT_RESUMED:" +PENDING_ROOT = "TestOpenAIAgentPendingObservation" +PENDING_CASES = ( + "pending_get_deadline", "pending_get_cancel", "latest_pending_get_deadline", "pending_timer_deadline", + "initial_backend_error", "initial_backend_deadline", "pending_backend_error", "pending_backend_deadline", + "completed", "failed", "conflicted", "missing", +) def contract(suite): + if suite == "pending": + leaves = {f"{PENDING_ROOT}/{case}" for case in PENDING_CASES} + red = {f"{PENDING_ROOT}/{case}" for case in ("pending_get_deadline", "pending_get_cancel", "latest_pending_get_deadline")} + return "service", leaves | {PENDING_ROOT}, red, leaves, set() if suite == "handler": roots = {POSITIVE, FENCES} parents = {f"{root}/{kind}" for root in roots for kind in KINDS} @@ -71,7 +81,7 @@ def validate(suite, phase, status, transcript): rows.append(row) counts = Counter((r.get("Action"), r.get("Test")) for r in rows) failures = set() - if suite == "handler" and phase == "RED": + if red_leaves and phase == "RED": for leaf in red_leaves: parts = leaf.split("/") failures.update("/".join(parts[:i]) for i in range(1, len(parts) + 1)) @@ -88,7 +98,11 @@ def validate(suite, phase, status, transcript): raise ValueError(f"wrong/missing/repeated outcome: {name}") for name in positives | fences: output = "".join(r.get("Output", "") for r in rows if r.get("Test") == name) - if name in failures: + if suite == "pending": + required = ("PENDING_OBSERVATION_LOST:", "owner_pending=1") if name in failures else ( + "PENDING_OBSERVATION_CONTROL_OK:", "classification preserved", "no ownership publication", + ) + elif name in failures: required = ( MARKER, "T1/T2 completed on A", "T2 omitted model", "O1/O2 and C delivered", "proof=failed", "T3 tool_provenance_unknown", "A disabled", "B sends=0", "records unchanged", diff --git a/.github/workflows/ws-inherited-model.yml b/.github/workflows/ws-inherited-model.yml index f2968987136f..bc5a8d953a05 100644 --- a/.github/workflows/ws-inherited-model.yml +++ b/.github/workflows/ws-inherited-model.yml @@ -96,3 +96,81 @@ jobs: name: ws-inherited-model-${{ matrix.phase }}-${{ env.CANDIDATE }} path: ${{ env.EVIDENCE }} if-no-files-found: error + pending-observation: + name: Pending observation ${{ matrix.phase }} + runs-on: ubuntu-24.04 + timeout-minutes: 25 + strategy: + fail-fast: false + matrix: + phase: [RED, GREEN] + env: + CANDIDATE: ${{ github.event.pull_request.head.sha || github.sha }} + PHASE: ${{ matrix.phase }} + PENDING_BASELINE: c213c632e428423ecc271ace88743f7b1d64b48e + steps: + - name: Initialize pending evidence directory + shell: bash + run: | + set -euo pipefail + export EVIDENCE="$RUNNER_TEMP/ws-pending-observation-evidence" + echo "EVIDENCE=$EVIDENCE" >> "$GITHUB_ENV" + mkdir -p "$EVIDENCE" + - uses: actions/checkout@v6 + with: + ref: ${{ env.CANDIDATE }} + fetch-depth: 0 + persist-credentials: false + - uses: actions/setup-go@v6 + with: + go-version-file: backend/go.mod + cache-dependency-path: backend/go.sum + - name: Restore only the pinned pending-wait implementation for RED + shell: bash + run: | + set -euo pipefail + test "$(git rev-parse HEAD)" = "$CANDIDATE" + git diff --exit-code + git fetch --no-tags --depth=1 origin "$PENDING_BASELINE" + target=backend/internal/service/openai_agent_message_delivery.go + fixture=backend/internal/service/openai_agent_pending_observation_test.go + if [ "$PHASE" = RED ]; then + git show "$PENDING_BASELINE:$target" > "$target" + test "$(git hash-object "$target")" = "$(git rev-parse "$PENDING_BASELINE:$target")" + test "$(git diff --name-only)" = "$target" + fi + test "$(git hash-object "$fixture")" = "$(git rev-parse "$CANDIDATE:$fixture")" + python3 - <<'PY' + import json, os, pathlib, subprocess + def git(*args): + return subprocess.check_output(['git', *args], text=True).strip() + target = 'backend/internal/service/openai_agent_message_delivery.go' + fixture = 'backend/internal/service/openai_agent_pending_observation_test.go' + result = { + 'candidate': git('rev-parse', 'HEAD'), 'baseline': os.environ['PENDING_BASELINE'], + 'phase': os.environ['PHASE'], 'test_blob': git('hash-object', fixture), + 'old_production_blob': git('rev-parse', os.environ['PENDING_BASELINE'] + ':' + target), + 'actual_production_blob': git('hash-object', target), + 'candidate_production_blob': git('rev-parse', 'HEAD:' + target), + 'changed_paths': git('diff', '--name-only').splitlines(), + } + (pathlib.Path(os.environ['EVIDENCE']) / 'source-manifest.json').write_text(json.dumps(result, indent=2) + '\n') + PY + python3 -B -m unittest discover -s .github/scripts -p 'test_ws_inherited_model_evidence.py' -v > "$EVIDENCE/parser-tests.log" 2>&1 + - name: Deterministic pending deadline and independent backend error controls + shell: bash + run: | + set -euo pipefail + set +e + (cd backend && go test -race -p=2 -json -tags=unit,inheritedmodel ./internal/service -run '^TestOpenAIAgentPendingObservation$' -count=1 -timeout=3m) > "$EVIDENCE/pending.jsonl" 2>&1 + status=$? + set -e + printf '%s\n' "$status" > "$EVIDENCE/pending.exit" + cat "$EVIDENCE/pending.jsonl" + python3 .github/scripts/ws-inherited-model-evidence.py pending "$PHASE" "$status" "$EVIDENCE/pending.jsonl" | tee "$EVIDENCE/pending-verdict.json" + - uses: actions/upload-artifact@v7 + if: always() + with: + name: ws-pending-observation-${{ matrix.phase }}-${{ env.CANDIDATE }} + path: ${{ env.EVIDENCE }} + if-no-files-found: error diff --git a/backend/internal/service/openai_agent_message_delivery.go b/backend/internal/service/openai_agent_message_delivery.go index ac2dba2f7716..fa3cf8d9438f 100644 --- a/backend/internal/service/openai_agent_message_delivery.go +++ b/backend/internal/service/openai_agent_message_delivery.go @@ -59,6 +59,7 @@ func newOpenAIAgentRecoveryDelivery(s *OpenAIGatewayService, ctx context.Context func (s *OpenAIGatewayService) awaitOpenAIAgentRecovery(ctx context.Context, shared OpenAIAgentRecoveryCache, state *openAIAgentRecoveryState, key [32]byte) (openAIAgentRecoveryBinding, bool, error) { observedPending := false + var lastPending openAIAgentRecoveryBinding for { var b openAIAgentRecoveryBinding var found bool @@ -77,7 +78,10 @@ func (s *OpenAIGatewayService) awaitOpenAIAgentRecovery(ctx context.Context, sha } if err != nil { if observedPending && ctx.Err() != nil { - return b, false, openAIAgentRecoveryError("delivery_pending") + // This Get may return no record when the same wait context + // expires. Retain the observed fence for diagnostics, while + // found=false and the error still prohibit an ownership grant. + return lastPending, false, openAIAgentRecoveryError("delivery_pending") } return b, false, openAIAgentRecoveryError("store_unavailable") } @@ -94,6 +98,7 @@ func (s *OpenAIGatewayService) awaitOpenAIAgentRecovery(ctx context.Context, sha return b, found, nil } observedPending = true + lastPending = b timer := time.NewTimer(5 * time.Millisecond) select { case <-ctx.Done(): diff --git a/backend/internal/service/openai_agent_pending_observation_test.go b/backend/internal/service/openai_agent_pending_observation_test.go new file mode 100644 index 000000000000..78de7509ca3e --- /dev/null +++ b/backend/internal/service/openai_agent_pending_observation_test.go @@ -0,0 +1,160 @@ +//go:build unit && inheritedmodel + +package service + +import ( + "context" + "errors" + "testing" + "testing/synctest" + "time" + + "github.com/stretchr/testify/require" +) + +type pendingObservationCache struct { + get func(context.Context, int) (string, error) + reads int + publishes int +} + +func (c *pendingObservationCache) GetOpenAIAgentRecovery(ctx context.Context, _ string) (string, error) { + c.reads++ + return c.get(ctx, c.reads) +} + +func (c *pendingObservationCache) PublishOpenAIAgentRecovery(context.Context, string, string, time.Duration) error { + c.publishes++ + return nil +} + +// Fake time advances only when the synctest bubble is durably blocked. The +// cache either blocks inside Get until THIS wait context ends, or returns an +// independent error while that context is live. No sleeps, timing tolerance, +// Redis scheduling luck, or diagnostic data granting authorization. +func TestOpenAIAgentPendingObservation(t *testing.T) { + for _, scenario := range []string{ + "pending_get_deadline", "pending_get_cancel", "latest_pending_get_deadline", "pending_timer_deadline", + "initial_backend_error", "initial_backend_deadline", "pending_backend_error", "pending_backend_deadline", + "completed", "failed", "conflicted", "missing", + } { + t.Run(scenario, func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + svc := &OpenAIGatewayService{} + producer := openAIAgentRecoveryBinding{accountID: 41, config: [32]byte{1}, wire: [32]byte{2}} + route := encodeOpenAIAgentRecoveryBinding(producer) + pending := "pending:fixture-first:" + route + latest := "pending:fixture-latest:" + route + waitFor := 20 * time.Millisecond + if scenario == "pending_timer_deadline" { + waitFor = time.Millisecond // strictly before the production poll timer + } + ctx, cancel := context.WithTimeout(context.Background(), waitFor) + defer cancel() + backendErr := errors.New("fixture independent cache failure") + cache := &pendingObservationCache{} + cache.get = func(got context.Context, read int) (string, error) { + require.Same(t, ctx, got, "the cache must receive the exact wait context") + if scenario == "initial_backend_error" { + return "", backendErr + } + if scenario == "initial_backend_deadline" { + return "", context.DeadlineExceeded // backend's deadline, not this live context + } + if read == 1 { + return pending, nil + } + switch scenario { + case "latest_pending_get_deadline": + if read == 2 { + return latest, nil + } + fallthrough + case "pending_get_deadline": + <-got.Done() + return "", got.Err() + case "pending_get_cancel": + cancel() // synchronous cancellation during the second Get + return "", got.Err() + case "pending_backend_error": + return "", backendErr + case "pending_backend_deadline": + return "", context.DeadlineExceeded + case "completed": + return route, nil + case "failed": + return "failed", nil + case "conflicted": + return "conflict", nil + case "missing": + return "", nil + default: + t.Fatal("timer branch must end before a second cache read") + return "", backendErr + } + } + binding, found, err := svc.awaitOpenAIAgentRecovery(ctx, cache, &svc.agentMessageRecovery, [32]byte{7}) + require.Zero(t, cache.publishes, "waiting never writes an ownership grant") + counts := openAIRecoveryOwnerCounts{observed: true, total: 3, completed: 2, unexamined: 1} + counts.observe(binding, found, err) + reason := func(want string) { + t.Helper() + var recovery *OpenAIAgentRecoveryError + require.ErrorAs(t, err, &recovery) + require.Equal(t, want, recovery.Reason) + require.False(t, found, "diagnostic evidence never authorizes a completed owner") + } + switch scenario { + case "pending_get_deadline", "pending_get_cancel", "latest_pending_get_deadline", "pending_timer_deadline": + reason("delivery_pending") + require.Error(t, ctx.Err()) + reads := 2 + wantPending := pending + if scenario == "pending_timer_deadline" { + reads = 1 + } + if scenario == "latest_pending_get_deadline" { + reads, wantPending = 3, latest + } + require.Equal(t, reads, cache.reads) + require.Equal(t, 2, counts.completed) + require.Zero(t, counts.unexamined) + require.Equal(t, 1, counts.pending, "PENDING_OBSERVATION_LOST: wait context ended after observed pending; refusal must retain owner_pending=1") + require.Zero(t, counts.unknown) + require.Equal(t, wantPending, binding.pending) + require.Equal(t, route, encodeOpenAIAgentRecoveryBinding(binding)) + case "initial_backend_error", "initial_backend_deadline", "pending_backend_error", "pending_backend_deadline": + reason("store_unavailable") + require.NoError(t, ctx.Err(), "independent backend errors must not be reclassified as wait expiry") + require.Empty(t, binding.pending) + require.Zero(t, counts.pending) + require.Equal(t, 1, counts.unknown) + reads := 2 + if scenario == "initial_backend_error" || scenario == "initial_backend_deadline" { + reads = 1 + } + require.Equal(t, reads, cache.reads) + case "completed": + require.NoError(t, err) + require.True(t, found) + require.Empty(t, binding.pending) + require.Equal(t, route, encodeOpenAIAgentRecoveryBinding(binding)) + require.Equal(t, 3, counts.completed) + case "failed": + reason("owner_unavailable") + require.True(t, binding.failed) + require.Equal(t, 1, counts.failed) + case "conflicted": + require.NoError(t, err) + require.True(t, found) + require.True(t, binding.conflicted, "caller retains its existing conflict rejection") + require.Equal(t, 1, counts.conflicted) + case "missing": + reason("owner_unavailable") + require.Equal(t, 1, counts.unknown) + } + t.Log("PENDING_OBSERVATION_CONTROL_OK: classification preserved; no ownership publication") + }) + }) + } +}