Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
71 changes: 71 additions & 0 deletions .github/scripts/pending-storage-error-evidence.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
"""Require the exact pending/storage behavioral failures, never build-only RED."""

import importlib.util
import json
import sys
from collections import Counter
from pathlib import Path

SPEC = importlib.util.spec_from_file_location(
"clean", Path(__file__).with_name("assert-readable-continuation.py")
)
CLEAN = importlib.util.module_from_spec(SPEC)
SPEC.loader.exec_module(CLEAN)

ROOT = "TestOpenAIAgentPendingStorageError"
CASES = (
"own_deadline", "own_cancel", "wrapped_own_deadline", "wrapped_own_cancel",
"backend_error_at_deadline", "backend_error_at_cancel",
"backend_deadline_at_cancel", "backend_cancel_at_deadline",
"backend_error_while_live", "backend_deadline_while_live",
)
RED_CASES = {
"backend_error_at_deadline", "backend_error_at_cancel",
"backend_deadline_at_cancel", "backend_cancel_at_deadline",
}
PACKAGE = "github.com/Wei-Shaw/sub2api/internal/service"


def validate(phase, status, transcript):
if phase not in {"RED", "GREEN"}:
raise ValueError("unknown phase")
CLEAN.check_clean(transcript)
tests = {ROOT} | {f"{ROOT}/{case}" for case in CASES}
failed = {f"{ROOT}/{case}" for case in RED_CASES} if phase == "RED" else set()
failures = failed | {ROOT, None} if failed else set()
if status != (1 if failures else 0):
raise ValueError("wrong process exit status")
rows = []
for line in transcript.splitlines():
if line.startswith("go: downloading "):
continue
row = json.loads(line)
if not isinstance(row, dict) or row.get("Package") != PACKAGE:
raise ValueError("unexpected package or malformed event")
if row.get("Action") not in {"start", "run", "output", "pass", "fail"}:
raise ValueError("unexpected action, including skip/build failure")
if row.get("Test") not in tests | {None}:
raise ValueError("unrelated test")
rows.append(row)
counts = Counter((r.get("Action"), r.get("Test")) for r in rows)
if counts["start", None] != 1:
raise ValueError("missing/repeated package start")
for name in tests | {None}:
if name is not None and counts["run", name] != 1:
raise ValueError(f"missing/repeated test run: {name}")
expected = "fail" if name in failures else "pass"
if counts[expected, name] != 1 or counts["pass", name] + counts["fail", name] != 1:
raise ValueError(f"wrong/missing/repeated outcome: {name}")
for name in tests - {ROOT}:
output = "".join(r.get("Output", "") for r in rows if r.get("Test") == name)
required = ("PENDING_STORAGE_ERROR_MASKED:", "store_unavailable", "delivery_pending") if name in failed else (
"PENDING_STORAGE_ERROR_CONTROL_OK:", "found=false", "no ownership publication",
)
if any(marker not in output for marker in required):
raise ValueError(f"missing behavioral evidence: {name}")
return {"phase": phase, "exit_status": status, "tests": len(tests), "behavioral_red_leaves": len(failed)}


if __name__ == "__main__":
phase, status, path = sys.argv[1:]
print(json.dumps(validate(phase, int(status), Path(path).read_text()), sort_keys=True))
124 changes: 124 additions & 0 deletions .github/workflows/recovery-pending-storage-error.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,124 @@
name: Recovery pending storage error

on:
pull_request:
branches: [production]
paths:
- 'backend/internal/service/openai*'
- '.github/workflows/recovery-pending-storage-error.yml'
- '.github/scripts/pending-storage-error-evidence.py'
- '.github/scripts/assert-readable-continuation.py'
workflow_dispatch:

permissions:
contents: read

jobs:
pending-storage-error:
name: Pending storage error RED then GREEN
runs-on: ubuntu-24.04
timeout-minutes: 30
env:
CANDIDATE: ${{ github.event.pull_request.head.sha || github.sha }}
BASELINE: dc3a5403be7053397c43c2ef4b1e4fff01536433
PYTHONDONTWRITEBYTECODE: '1'
steps:
- uses: actions/checkout@v6
with:
ref: ${{ env.CANDIDATE }}
fetch-depth: 0
persist-credentials: false
- uses: actions/setup-go@v6
with:
go-version-file: backend/go.mod
cache-dependency-path: backend/go.sum
- name: Isolate exact old implementation with identical candidate fixtures
shell: bash
run: |
set -euo pipefail
evidence_dir="$RUNNER_TEMP/pending-storage-error-evidence"
red_dir="$RUNNER_TEMP/pending-storage-error-red"
mkdir -p "$evidence_dir"
echo "EVIDENCE=$evidence_dir" >> "$GITHUB_ENV"
echo "RED_ROOT=$red_dir" >> "$GITHUB_ENV"
test "$(git rev-parse HEAD)" = "$CANDIDATE"
test -z "$(git status --porcelain)"
git fetch --no-tags --depth=1 origin "$BASELINE"
git worktree add --detach "$red_dir" "$CANDIDATE"
target=backend/internal/service/openai_agent_message_delivery.go
git show "$BASELINE:$target" > "$red_dir/$target"
test "$(git hash-object "$red_dir/$target")" = "$(git rev-parse "$BASELINE:$target")"
test "$(git -C "$red_dir" diff --name-only)" = "$target"
for fixture in openai_agent_pending_storage_error_test.go openai_agent_pending_observation_test.go openai_ws_owner_conflict_test.go; do
path="backend/internal/service/$fixture"
test "$(git hash-object "$red_dir/$path")" = "$(git rev-parse "$CANDIDATE:$path")"
done
python3 - <<'PY'
import json, os, pathlib, subprocess
def git(*args):
return subprocess.check_output(['git', *args], text=True).strip()
root = os.environ['RUNNER_TEMP'] + '/pending-storage-error-red'
target = 'backend/internal/service/openai_agent_message_delivery.go'
fixtures = ['backend/internal/service/' + name for name in (
'openai_agent_pending_storage_error_test.go', 'openai_agent_pending_observation_test.go',
'openai_ws_owner_conflict_test.go')]
manifest = {
'candidate': git('rev-parse', 'HEAD'), 'baseline': os.environ['BASELINE'],
'baseline_production_blob': git('rev-parse', os.environ['BASELINE'] + ':' + target),
'red_production_blob': git('hash-object', root + '/' + target),
'green_production_blob': git('rev-parse', 'HEAD:' + target),
'identical_candidate_fixtures': {path: git('hash-object', root + '/' + path) for path in fixtures},
'red_changed_paths': git('-C', root, 'diff', '--name-only').splitlines(),
}
destination = pathlib.Path(os.environ['RUNNER_TEMP']) / 'pending-storage-error-evidence'
(destination / 'source-manifest.json').write_text(json.dumps(manifest, indent=2) + '\n')
PY
- name: Behavioral RED on old code
shell: bash
run: |
set -euo pipefail
set +e
(cd "$RED_ROOT/backend" && go test -race -p=2 -json -tags=unit,inheritedmodel ./internal/service -run '^TestOpenAIAgentPendingStorageError$' -count=1 -timeout=3m) > "$EVIDENCE/red.jsonl" 2>&1
status=$?
set -e
printf '%s\n' "$status" > "$EVIDENCE/red.exit"
cat "$EVIDENCE/red.jsonl"
python3 .github/scripts/pending-storage-error-evidence.py RED "$status" "$EVIDENCE/red.jsonl" | tee "$EVIDENCE/red-verdict.json"
- name: Behavioral GREEN with the same regression tests
shell: bash
run: |
set -euo pipefail
test "$(git rev-parse HEAD)" = "$CANDIDATE"
test -z "$(git status --porcelain)"
set +e
(cd backend && go test -race -p=2 -json -tags=unit,inheritedmodel ./internal/service -run '^TestOpenAIAgentPendingStorageError$' -count=1 -timeout=3m) > "$EVIDENCE/green.jsonl" 2>&1
status=$?
set -e
printf '%s\n' "$status" > "$EVIDENCE/green.exit"
cat "$EVIDENCE/green.jsonl"
python3 .github/scripts/pending-storage-error-evidence.py GREEN "$status" "$EVIDENCE/green.jsonl" | tee "$EVIDENCE/green-verdict.json"
- name: Preserve original pending and tool-proof refusal guards
shell: bash
run: |
set -euo pipefail
python3 -B -m unittest discover -s .github/scripts -p 'test_ws_inherited_model_evidence.py' -v > "$EVIDENCE/parser-tests.log" 2>&1
(cd backend && go test -race -p=2 -json -tags=unit,inheritedmodel ./internal/service -run '^TestOpenAIAgentPendingObservation$' -count=1 -timeout=3m) > "$EVIDENCE/pending-guards.jsonl" 2>&1
python3 .github/scripts/ws-inherited-model-evidence.py pending GREEN 0 "$EVIDENCE/pending-guards.jsonl" | tee "$EVIDENCE/pending-guards-verdict.json"
(cd backend && go test -race -p=2 -json -tags=unit,ownerconflict ./internal/service -run '^TestOpenAIToolProvenancePendingDeadline$' -count=1 -timeout=3m) > "$EVIDENCE/tool-guards.jsonl" 2>&1
python3 .github/scripts/assert-readable-continuation.py --clean-log "$EVIDENCE/tool-guards.jsonl"
python3 - <<'PY'
import json, os, pathlib
rows = [json.loads(line) for line in (pathlib.Path(os.environ['EVIDENCE']) / 'tool-guards.jsonl').read_text().splitlines() if line.startswith('{')]
for kind in ('function', 'custom'):
for case in ('pending_deadline', 'initial_deadline', 'initial_store_failure', 'initial_backend_deadline', 'pending_store_failure', 'pending_backend_deadline', 'pending_store_failure_at_deadline'):
name = f'TestOpenAIToolProvenancePendingDeadline/{kind}/{case}'
assert sum(row.get('Action') == 'pass' and row.get('Test') == name for row in rows) == 1, name
assert not any(row.get('Action') in ('fail', 'skip') for row in rows)
PY
test -z "$(git status --porcelain)"
- uses: actions/upload-artifact@v7
if: always()
with:
name: pending-storage-error-${{ env.CANDIDATE }}
path: ${{ runner.temp }}/pending-storage-error-evidence
if-no-files-found: error
17 changes: 15 additions & 2 deletions backend/internal/mihomo/sources_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -37,8 +37,21 @@ func cachedSource(nodeName string) subscriptionCache {

func waitSourceOperation(t *testing.T, m *Manager) Status {
t.Helper()
require.Eventually(t, func() bool { return !m.Status().Busy }, time.Second*3, time.Millisecond*5)
return m.Status()
// Wait for the submitted operation itself. Repeated Status snapshots copy
// large proxy lists and compete with config generation on busy CI runners.
done := make(chan struct{})
go func() {
m.wg.Wait()
close(done)
}()
select {
case <-done:
case <-time.After(30 * time.Second):
t.Fatal("source operation did not complete within 30 seconds")
}
status := m.Status()
require.False(t, status.Busy)
return status
}

func TestSourceRemovalPreservesCachedPeersDynamicAndSharedNodes(t *testing.T) {
Expand Down
9 changes: 5 additions & 4 deletions backend/internal/service/openai_agent_message_delivery.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package service
import (
"context"
"encoding/hex"
"errors"
"strings"
"sync"
"time"
Expand Down Expand Up @@ -77,10 +78,10 @@ func (s *OpenAIGatewayService) awaitOpenAIAgentRecovery(ctx context.Context, sha
found = found && time.Now().Before(b.expiresAt)
}
if err != nil {
if observedPending && ctx.Err() != nil {
// This Get may return no record when the same wait context
// expires. Retain the observed fence for diagnostics, while
// found=false and the error still prohibit an ownership grant.
if observedPending && ctx.Err() != nil && errors.Is(err, ctx.Err()) {
// Retain pending diagnostics only for the wait's context error.
// Independent storage failures remain unavailable; found=false
// and the error still prohibit an ownership grant.
return lastPending, false, openAIAgentRecoveryError("delivery_pending")
}
return b, false, openAIAgentRecoveryError("store_unavailable")
Expand Down
100 changes: 100 additions & 0 deletions backend/internal/service/openai_agent_pending_storage_error_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,100 @@
//go:build unit && inheritedmodel

package service

import (
"context"
"errors"
"fmt"
"testing"
"testing/synctest"
"time"

"github.com/stretchr/testify/require"
)

// End the wait inside the second GET, after pending was observed. This avoids
// racing the polling timer and proves which error the GET actually returned.
func TestOpenAIAgentPendingStorageError(t *testing.T) {
backendErr := errors.New("fixture independent cache failure")
for _, tc := range []struct {
name string
end string
readErr error
own, wrap bool
wantPending bool
}{
{"own_deadline", "deadline", nil, true, false, true},
{"own_cancel", "cancel", nil, true, false, true},
{"wrapped_own_deadline", "deadline", nil, true, true, true},
{"wrapped_own_cancel", "cancel", nil, true, true, true},
{"backend_error_at_deadline", "deadline", backendErr, false, false, false},
{"backend_error_at_cancel", "cancel", backendErr, false, false, false},
{"backend_deadline_at_cancel", "cancel", context.DeadlineExceeded, false, true, false},
{"backend_cancel_at_deadline", "deadline", context.Canceled, false, true, false},
{"backend_error_while_live", "", backendErr, false, false, false},
{"backend_deadline_while_live", "", context.DeadlineExceeded, false, true, false},
} {
t.Run(tc.name, func(t *testing.T) {
synctest.Test(t, func(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond)
defer cancel()
svc := &OpenAIGatewayService{}
producer := openAIAgentRecoveryBinding{accountID: 41, config: [32]byte{1}, wire: [32]byte{2}}
route := encodeOpenAIAgentRecoveryBinding(producer)
pending := "pending:fixture-storage-error:" + route
var returnedErr error
cache := &pendingObservationCache{get: func(got context.Context, read int) (string, error) {
require.Same(t, ctx, got)
if read == 1 {
return pending, nil
}
require.Equal(t, 2, read, "a storage error must end the wait without replay")
switch tc.end {
case "deadline":
<-got.Done()
case "cancel":
cancel()
}
returnedErr = tc.readErr
if tc.own {
returnedErr = got.Err()
}
if tc.wrap {
returnedErr = fmt.Errorf("fixture cache GET: %w", returnedErr)
}
return "", returnedErr
}}
binding, found, err := svc.awaitOpenAIAgentRecovery(ctx, cache, &svc.agentMessageRecovery, [32]byte{7})
require.Equal(t, 2, cache.reads)
require.Zero(t, cache.publishes, "waiting never publishes an ownership grant")
require.False(t, found, "diagnostics must never authorize an owner")
if tc.end == "" {
require.NoError(t, ctx.Err())
} else {
require.Error(t, ctx.Err(), "the wait must end before the GET returns")
}
require.Equal(t, tc.own, ctx.Err() != nil && errors.Is(returnedErr, ctx.Err()))
var recovery *OpenAIAgentRecoveryError
require.ErrorAs(t, err, &recovery)
counts := openAIRecoveryOwnerCounts{observed: true, total: 3, completed: 2, unexamined: 1}
counts.observe(binding, found, err)
if tc.wantPending {
require.Equal(t, "delivery_pending", recovery.Reason)
require.Equal(t, pending, binding.pending)
require.Equal(t, route, encodeOpenAIAgentRecoveryBinding(binding))
require.Equal(t, 1, counts.pending)
require.Zero(t, counts.unknown)
} else {
require.Equal(t, "store_unavailable", recovery.Reason, "PENDING_STORAGE_ERROR_MASKED: independent GET failure must remain store_unavailable")
require.Empty(t, binding.pending)
require.Zero(t, counts.pending)
require.Equal(t, 1, counts.unknown)
}
require.Equal(t, 2, counts.completed)
require.Zero(t, counts.unexamined)
t.Log("PENDING_STORAGE_ERROR_CONTROL_OK: classification preserved; found=false; no ownership publication")
})
})
}
}