diff --git a/.github/scripts/pending-storage-error-evidence.py b/.github/scripts/pending-storage-error-evidence.py new file mode 100644 index 000000000000..c9de4c6a7f93 --- /dev/null +++ b/.github/scripts/pending-storage-error-evidence.py @@ -0,0 +1,71 @@ +"""Require the exact pending/storage behavioral failures, never build-only RED.""" + +import importlib.util +import json +import sys +from collections import Counter +from pathlib import Path + +SPEC = importlib.util.spec_from_file_location( + "clean", Path(__file__).with_name("assert-readable-continuation.py") +) +CLEAN = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(CLEAN) + +ROOT = "TestOpenAIAgentPendingStorageError" +CASES = ( + "own_deadline", "own_cancel", "wrapped_own_deadline", "wrapped_own_cancel", + "backend_error_at_deadline", "backend_error_at_cancel", + "backend_deadline_at_cancel", "backend_cancel_at_deadline", + "backend_error_while_live", "backend_deadline_while_live", +) +RED_CASES = { + "backend_error_at_deadline", "backend_error_at_cancel", + "backend_deadline_at_cancel", "backend_cancel_at_deadline", +} +PACKAGE = "github.com/Wei-Shaw/sub2api/internal/service" + + +def validate(phase, status, transcript): + if phase not in {"RED", "GREEN"}: + raise ValueError("unknown phase") + CLEAN.check_clean(transcript) + tests = {ROOT} | {f"{ROOT}/{case}" for case in CASES} + failed = {f"{ROOT}/{case}" for case in RED_CASES} if phase == "RED" else set() + failures = failed | {ROOT, None} if failed else set() + if status != (1 if failures else 0): + raise ValueError("wrong process exit status") + rows = [] + for line in transcript.splitlines(): + if line.startswith("go: downloading "): + continue + row = json.loads(line) + if not isinstance(row, dict) or row.get("Package") != PACKAGE: + raise ValueError("unexpected package or malformed event") + if row.get("Action") not in {"start", "run", "output", "pass", "fail"}: + raise ValueError("unexpected action, including skip/build failure") + if row.get("Test") not in tests | {None}: + raise ValueError("unrelated test") + rows.append(row) + counts = Counter((r.get("Action"), r.get("Test")) for r in rows) + if counts["start", None] != 1: + raise ValueError("missing/repeated package start") + for name in tests | {None}: + if name is not None and counts["run", name] != 1: + raise ValueError(f"missing/repeated test run: {name}") + expected = "fail" if name in failures else "pass" + if counts[expected, name] != 1 or counts["pass", name] + counts["fail", name] != 1: + raise ValueError(f"wrong/missing/repeated outcome: {name}") + for name in tests - {ROOT}: + output = "".join(r.get("Output", "") for r in rows if r.get("Test") == name) + required = ("PENDING_STORAGE_ERROR_MASKED:", "store_unavailable", "delivery_pending") if name in failed else ( + "PENDING_STORAGE_ERROR_CONTROL_OK:", "found=false", "no ownership publication", + ) + if any(marker not in output for marker in required): + raise ValueError(f"missing behavioral evidence: {name}") + return {"phase": phase, "exit_status": status, "tests": len(tests), "behavioral_red_leaves": len(failed)} + + +if __name__ == "__main__": + phase, status, path = sys.argv[1:] + print(json.dumps(validate(phase, int(status), Path(path).read_text()), sort_keys=True)) diff --git a/.github/workflows/recovery-pending-storage-error.yml b/.github/workflows/recovery-pending-storage-error.yml new file mode 100644 index 000000000000..73cdd1f9957a --- /dev/null +++ b/.github/workflows/recovery-pending-storage-error.yml @@ -0,0 +1,124 @@ +name: Recovery pending storage error + +on: + pull_request: + branches: [production] + paths: + - 'backend/internal/service/openai*' + - '.github/workflows/recovery-pending-storage-error.yml' + - '.github/scripts/pending-storage-error-evidence.py' + - '.github/scripts/assert-readable-continuation.py' + workflow_dispatch: + +permissions: + contents: read + +jobs: + pending-storage-error: + name: Pending storage error RED then GREEN + runs-on: ubuntu-24.04 + timeout-minutes: 30 + env: + CANDIDATE: ${{ github.event.pull_request.head.sha || github.sha }} + BASELINE: dc3a5403be7053397c43c2ef4b1e4fff01536433 + PYTHONDONTWRITEBYTECODE: '1' + steps: + - uses: actions/checkout@v6 + with: + ref: ${{ env.CANDIDATE }} + fetch-depth: 0 + persist-credentials: false + - uses: actions/setup-go@v6 + with: + go-version-file: backend/go.mod + cache-dependency-path: backend/go.sum + - name: Isolate exact old implementation with identical candidate fixtures + shell: bash + run: | + set -euo pipefail + evidence_dir="$RUNNER_TEMP/pending-storage-error-evidence" + red_dir="$RUNNER_TEMP/pending-storage-error-red" + mkdir -p "$evidence_dir" + echo "EVIDENCE=$evidence_dir" >> "$GITHUB_ENV" + echo "RED_ROOT=$red_dir" >> "$GITHUB_ENV" + test "$(git rev-parse HEAD)" = "$CANDIDATE" + test -z "$(git status --porcelain)" + git fetch --no-tags --depth=1 origin "$BASELINE" + git worktree add --detach "$red_dir" "$CANDIDATE" + target=backend/internal/service/openai_agent_message_delivery.go + git show "$BASELINE:$target" > "$red_dir/$target" + test "$(git hash-object "$red_dir/$target")" = "$(git rev-parse "$BASELINE:$target")" + test "$(git -C "$red_dir" diff --name-only)" = "$target" + for fixture in openai_agent_pending_storage_error_test.go openai_agent_pending_observation_test.go openai_ws_owner_conflict_test.go; do + path="backend/internal/service/$fixture" + test "$(git hash-object "$red_dir/$path")" = "$(git rev-parse "$CANDIDATE:$path")" + done + python3 - <<'PY' + import json, os, pathlib, subprocess + def git(*args): + return subprocess.check_output(['git', *args], text=True).strip() + root = os.environ['RUNNER_TEMP'] + '/pending-storage-error-red' + target = 'backend/internal/service/openai_agent_message_delivery.go' + fixtures = ['backend/internal/service/' + name for name in ( + 'openai_agent_pending_storage_error_test.go', 'openai_agent_pending_observation_test.go', + 'openai_ws_owner_conflict_test.go')] + manifest = { + 'candidate': git('rev-parse', 'HEAD'), 'baseline': os.environ['BASELINE'], + 'baseline_production_blob': git('rev-parse', os.environ['BASELINE'] + ':' + target), + 'red_production_blob': git('hash-object', root + '/' + target), + 'green_production_blob': git('rev-parse', 'HEAD:' + target), + 'identical_candidate_fixtures': {path: git('hash-object', root + '/' + path) for path in fixtures}, + 'red_changed_paths': git('-C', root, 'diff', '--name-only').splitlines(), + } + destination = pathlib.Path(os.environ['RUNNER_TEMP']) / 'pending-storage-error-evidence' + (destination / 'source-manifest.json').write_text(json.dumps(manifest, indent=2) + '\n') + PY + - name: Behavioral RED on old code + shell: bash + run: | + set -euo pipefail + set +e + (cd "$RED_ROOT/backend" && go test -race -p=2 -json -tags=unit,inheritedmodel ./internal/service -run '^TestOpenAIAgentPendingStorageError$' -count=1 -timeout=3m) > "$EVIDENCE/red.jsonl" 2>&1 + status=$? + set -e + printf '%s\n' "$status" > "$EVIDENCE/red.exit" + cat "$EVIDENCE/red.jsonl" + python3 .github/scripts/pending-storage-error-evidence.py RED "$status" "$EVIDENCE/red.jsonl" | tee "$EVIDENCE/red-verdict.json" + - name: Behavioral GREEN with the same regression tests + shell: bash + run: | + set -euo pipefail + test "$(git rev-parse HEAD)" = "$CANDIDATE" + test -z "$(git status --porcelain)" + set +e + (cd backend && go test -race -p=2 -json -tags=unit,inheritedmodel ./internal/service -run '^TestOpenAIAgentPendingStorageError$' -count=1 -timeout=3m) > "$EVIDENCE/green.jsonl" 2>&1 + status=$? + set -e + printf '%s\n' "$status" > "$EVIDENCE/green.exit" + cat "$EVIDENCE/green.jsonl" + python3 .github/scripts/pending-storage-error-evidence.py GREEN "$status" "$EVIDENCE/green.jsonl" | tee "$EVIDENCE/green-verdict.json" + - name: Preserve original pending and tool-proof refusal guards + shell: bash + run: | + set -euo pipefail + python3 -B -m unittest discover -s .github/scripts -p 'test_ws_inherited_model_evidence.py' -v > "$EVIDENCE/parser-tests.log" 2>&1 + (cd backend && go test -race -p=2 -json -tags=unit,inheritedmodel ./internal/service -run '^TestOpenAIAgentPendingObservation$' -count=1 -timeout=3m) > "$EVIDENCE/pending-guards.jsonl" 2>&1 + python3 .github/scripts/ws-inherited-model-evidence.py pending GREEN 0 "$EVIDENCE/pending-guards.jsonl" | tee "$EVIDENCE/pending-guards-verdict.json" + (cd backend && go test -race -p=2 -json -tags=unit,ownerconflict ./internal/service -run '^TestOpenAIToolProvenancePendingDeadline$' -count=1 -timeout=3m) > "$EVIDENCE/tool-guards.jsonl" 2>&1 + python3 .github/scripts/assert-readable-continuation.py --clean-log "$EVIDENCE/tool-guards.jsonl" + python3 - <<'PY' + import json, os, pathlib + rows = [json.loads(line) for line in (pathlib.Path(os.environ['EVIDENCE']) / 'tool-guards.jsonl').read_text().splitlines() if line.startswith('{')] + for kind in ('function', 'custom'): + for case in ('pending_deadline', 'initial_deadline', 'initial_store_failure', 'initial_backend_deadline', 'pending_store_failure', 'pending_backend_deadline', 'pending_store_failure_at_deadline'): + name = f'TestOpenAIToolProvenancePendingDeadline/{kind}/{case}' + assert sum(row.get('Action') == 'pass' and row.get('Test') == name for row in rows) == 1, name + assert not any(row.get('Action') in ('fail', 'skip') for row in rows) + PY + test -z "$(git status --porcelain)" + - uses: actions/upload-artifact@v7 + if: always() + with: + name: pending-storage-error-${{ env.CANDIDATE }} + path: ${{ runner.temp }}/pending-storage-error-evidence + if-no-files-found: error diff --git a/backend/internal/mihomo/sources_test.go b/backend/internal/mihomo/sources_test.go index 157a759c85a9..a81eb31c8316 100644 --- a/backend/internal/mihomo/sources_test.go +++ b/backend/internal/mihomo/sources_test.go @@ -37,8 +37,21 @@ func cachedSource(nodeName string) subscriptionCache { func waitSourceOperation(t *testing.T, m *Manager) Status { t.Helper() - require.Eventually(t, func() bool { return !m.Status().Busy }, time.Second*3, time.Millisecond*5) - return m.Status() + // Wait for the submitted operation itself. Repeated Status snapshots copy + // large proxy lists and compete with config generation on busy CI runners. + done := make(chan struct{}) + go func() { + m.wg.Wait() + close(done) + }() + select { + case <-done: + case <-time.After(30 * time.Second): + t.Fatal("source operation did not complete within 30 seconds") + } + status := m.Status() + require.False(t, status.Busy) + return status } func TestSourceRemovalPreservesCachedPeersDynamicAndSharedNodes(t *testing.T) { diff --git a/backend/internal/service/openai_agent_message_delivery.go b/backend/internal/service/openai_agent_message_delivery.go index fa3cf8d9438f..4fb4b14dc25f 100644 --- a/backend/internal/service/openai_agent_message_delivery.go +++ b/backend/internal/service/openai_agent_message_delivery.go @@ -3,6 +3,7 @@ package service import ( "context" "encoding/hex" + "errors" "strings" "sync" "time" @@ -77,10 +78,10 @@ func (s *OpenAIGatewayService) awaitOpenAIAgentRecovery(ctx context.Context, sha found = found && time.Now().Before(b.expiresAt) } if err != nil { - if observedPending && ctx.Err() != nil { - // This Get may return no record when the same wait context - // expires. Retain the observed fence for diagnostics, while - // found=false and the error still prohibit an ownership grant. + if observedPending && ctx.Err() != nil && errors.Is(err, ctx.Err()) { + // Retain pending diagnostics only for the wait's context error. + // Independent storage failures remain unavailable; found=false + // and the error still prohibit an ownership grant. return lastPending, false, openAIAgentRecoveryError("delivery_pending") } return b, false, openAIAgentRecoveryError("store_unavailable") diff --git a/backend/internal/service/openai_agent_pending_storage_error_test.go b/backend/internal/service/openai_agent_pending_storage_error_test.go new file mode 100644 index 000000000000..131ab2562ccb --- /dev/null +++ b/backend/internal/service/openai_agent_pending_storage_error_test.go @@ -0,0 +1,100 @@ +//go:build unit && inheritedmodel + +package service + +import ( + "context" + "errors" + "fmt" + "testing" + "testing/synctest" + "time" + + "github.com/stretchr/testify/require" +) + +// End the wait inside the second GET, after pending was observed. This avoids +// racing the polling timer and proves which error the GET actually returned. +func TestOpenAIAgentPendingStorageError(t *testing.T) { + backendErr := errors.New("fixture independent cache failure") + for _, tc := range []struct { + name string + end string + readErr error + own, wrap bool + wantPending bool + }{ + {"own_deadline", "deadline", nil, true, false, true}, + {"own_cancel", "cancel", nil, true, false, true}, + {"wrapped_own_deadline", "deadline", nil, true, true, true}, + {"wrapped_own_cancel", "cancel", nil, true, true, true}, + {"backend_error_at_deadline", "deadline", backendErr, false, false, false}, + {"backend_error_at_cancel", "cancel", backendErr, false, false, false}, + {"backend_deadline_at_cancel", "cancel", context.DeadlineExceeded, false, true, false}, + {"backend_cancel_at_deadline", "deadline", context.Canceled, false, true, false}, + {"backend_error_while_live", "", backendErr, false, false, false}, + {"backend_deadline_while_live", "", context.DeadlineExceeded, false, true, false}, + } { + t.Run(tc.name, func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond) + defer cancel() + svc := &OpenAIGatewayService{} + producer := openAIAgentRecoveryBinding{accountID: 41, config: [32]byte{1}, wire: [32]byte{2}} + route := encodeOpenAIAgentRecoveryBinding(producer) + pending := "pending:fixture-storage-error:" + route + var returnedErr error + cache := &pendingObservationCache{get: func(got context.Context, read int) (string, error) { + require.Same(t, ctx, got) + if read == 1 { + return pending, nil + } + require.Equal(t, 2, read, "a storage error must end the wait without replay") + switch tc.end { + case "deadline": + <-got.Done() + case "cancel": + cancel() + } + returnedErr = tc.readErr + if tc.own { + returnedErr = got.Err() + } + if tc.wrap { + returnedErr = fmt.Errorf("fixture cache GET: %w", returnedErr) + } + return "", returnedErr + }} + binding, found, err := svc.awaitOpenAIAgentRecovery(ctx, cache, &svc.agentMessageRecovery, [32]byte{7}) + require.Equal(t, 2, cache.reads) + require.Zero(t, cache.publishes, "waiting never publishes an ownership grant") + require.False(t, found, "diagnostics must never authorize an owner") + if tc.end == "" { + require.NoError(t, ctx.Err()) + } else { + require.Error(t, ctx.Err(), "the wait must end before the GET returns") + } + require.Equal(t, tc.own, ctx.Err() != nil && errors.Is(returnedErr, ctx.Err())) + var recovery *OpenAIAgentRecoveryError + require.ErrorAs(t, err, &recovery) + counts := openAIRecoveryOwnerCounts{observed: true, total: 3, completed: 2, unexamined: 1} + counts.observe(binding, found, err) + if tc.wantPending { + require.Equal(t, "delivery_pending", recovery.Reason) + require.Equal(t, pending, binding.pending) + require.Equal(t, route, encodeOpenAIAgentRecoveryBinding(binding)) + require.Equal(t, 1, counts.pending) + require.Zero(t, counts.unknown) + } else { + require.Equal(t, "store_unavailable", recovery.Reason, "PENDING_STORAGE_ERROR_MASKED: independent GET failure must remain store_unavailable") + require.Empty(t, binding.pending) + require.Zero(t, counts.pending) + require.Equal(t, 1, counts.unknown) + } + require.Equal(t, 2, counts.completed) + require.Zero(t, counts.unexamined) + t.Log("PENDING_STORAGE_ERROR_CONTROL_OK: classification preserved; found=false; no ownership publication") + }) + }) + } +}