Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 34 additions & 13 deletions .github/scripts/issue871-evidence.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
"""Accept only the named diagnosis assertion as the remote behavioral RED."""
"""Accept only the named diagnosis or business assertion as behavioral RED."""

import json
import sys
Expand All @@ -10,9 +10,27 @@
NODES = {ROOT, LEAF}
MARKER = "issue871_expected_search_diagnosis"
ASSERTION = 'does not contain "phase=search_http; reason=http_rejected"'
BUSINESS_ROOT = "TestExcelBPSSearchBusinessThroughForward"
BUSINESS_LEAF = BUSINESS_ROOT + "/direct/json"
BUSINESS_MARKER = "issue871_expected_native_search_history"
BUSINESS_ASSERTION = BUSINESS_MARKER + ': got "__sub2api_web_search"; want "run_officejs"'
PENDING_ROOT = "TestOpenAIToolProvenancePendingDeadline"
PENDING_LEAF = PENDING_ROOT + "/function/pending_deadline"
PENDING_MARKER = "WS_PENDING_DEADLINE_CLASSIFICATION"
PENDING_ASSERTION = PENDING_MARKER + ': got "encrypted_agent_recovery_store_unavailable"; want "encrypted_agent_recovery_owner_conflict"'


def validate(phase, status, raw, stderr=""):
def validate(phase, status, raw, stderr="", contract="diagnosis"):
if contract == "diagnosis":
nodes, leaf, marker, assertion = NODES, LEAF, MARKER, ASSERTION
elif contract == "business":
nodes = {BUSINESS_ROOT, BUSINESS_LEAF}
leaf, marker, assertion = BUSINESS_LEAF, BUSINESS_MARKER, BUSINESS_ASSERTION
elif contract == "pending_deadline":
nodes = {PENDING_ROOT, PENDING_LEAF}
leaf, marker, assertion = PENDING_LEAF, PENDING_MARKER, PENDING_ASSERTION
else:
raise ValueError("unknown evidence contract")
if phase not in {"RED", "GREEN"} or status != (1 if phase == "RED" else 0):
raise ValueError("unexpected phase or test exit status")
combined = (raw + stderr).lower()
Expand All @@ -24,8 +42,8 @@ def validate(phase, status, raw, stderr=""):
if forbidden in combined:
raise ValueError("invalid behavioral evidence: " + forbidden)
events = [json.loads(line) for line in raw.splitlines() if line.strip()]
runs = {name: 0 for name in NODES}
terminals = {name: [] for name in NODES}
runs = {name: 0 for name in nodes}
terminals = {name: [] for name in nodes}
packages = []
output = ""
for event in events:
Expand All @@ -35,26 +53,29 @@ def validate(phase, status, raw, stderr=""):
if action not in {"start", "run", "pause", "cont", "output", "pass", "fail"}:
raise ValueError("skip/build/unknown event")
if test:
if test not in NODES:
if test not in nodes:
raise ValueError("unexpected test: " + test)
if action == "run":
runs[test] += 1
if action in {"pass", "fail"}:
terminals[test].append(action)
if action == "output" and test == LEAF:
if action == "output" and test == leaf:
output += event.get("Output", "")
elif action in {"pass", "fail"}:
packages.append(action)
expected = "fail" if phase == "RED" else "pass"
if runs != {name: 1 for name in NODES}:
if runs != {name: 1 for name in nodes}:
raise ValueError("missing or repeated named test execution")
if terminals != {name: [expected] for name in NODES} or packages != [expected]:
if terminals != {name: [expected] for name in nodes} or packages != [expected]:
raise ValueError("unexpected test/package terminal")
if phase == "RED" and (output.count(MARKER) != 1 or ASSERTION not in output):
raise ValueError("RED did not fail at the intended diagnosis assertion")
if phase == "RED" and (output.count(marker) != 1 or assertion not in output):
raise ValueError("RED did not fail at the intended " + contract + " assertion")


if __name__ == "__main__":
phase, status, log, stderr = sys.argv[1:]
validate(phase, int(status), Path(log).read_text(), Path(stderr).read_text())
print(phase + " named search diagnosis evidence accepted")
phase, status, log, stderr, *extra = sys.argv[1:]
if len(extra) > 1:
raise ValueError("expected at most one evidence contract")
contract = extra[0] if extra else "diagnosis"
validate(phase, int(status), Path(log).read_text(), Path(stderr).read_text(), contract)
print(phase + " named search " + contract + " evidence accepted")
67 changes: 62 additions & 5 deletions .github/scripts/test_issue871_evidence.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,21 +8,78 @@
spec.loader.exec_module(gate)


def transcript(phase):
def transcript(phase, contract="diagnosis"):
root, leaf, marker, assertion = gate.ROOT, gate.LEAF, gate.MARKER, gate.ASSERTION
if contract == "business":
root, leaf = gate.BUSINESS_ROOT, gate.BUSINESS_LEAF
marker, assertion = "", gate.BUSINESS_ASSERTION
elif contract == "pending_deadline":
root, leaf = gate.PENDING_ROOT, gate.PENDING_LEAF
marker, assertion = "", gate.PENDING_ASSERTION
terminal = "fail" if phase == "RED" else "pass"
events = [dict(Action="start", Package=gate.PACKAGE)]
for name in (gate.ROOT, gate.LEAF):
for name in (root, leaf):
events.append(dict(Action="run", Package=gate.PACKAGE, Test=name))
if phase == "RED":
events.append(dict(Action="output", Package=gate.PACKAGE, Test=gate.LEAF,
Output=gate.MARKER + "\n" + gate.ASSERTION + "\n"))
for name in (gate.LEAF, gate.ROOT):
events.append(dict(Action="output", Package=gate.PACKAGE, Test=leaf,
Output=marker + "\n" + assertion + "\n"))
for name in (leaf, root):
events.append(dict(Action=terminal, Package=gate.PACKAGE, Test=name))
events.append(dict(Action=terminal, Package=gate.PACKAGE))
return "\n".join(json.dumps(event) for event in events)


class Issue871EvidenceTest(unittest.TestCase):
def test_pending_deadline_contract(self):
for phase, code in (("RED", 1), ("GREEN", 0)):
raw = transcript(phase, "pending_deadline")
gate.validate(phase, code, raw, contract="pending_deadline")
for noise in ("panic: fixture", "build failed", "setup failed", "no tests to run",
"DATA RACE", "--- SKIP: fixture", "test timed out", "fatal error"):
with self.subTest(phase=phase, noise=noise), self.assertRaises(ValueError):
gate.validate(phase, code, raw, noise, "pending_deadline")
for invalid in ("", raw + "\n" + raw, raw.replace(gate.PENDING_LEAF, gate.PENDING_ROOT + "/wrong"),
raw.replace('"Action": "run"', '"Action": "skip"', 1), transcript(phase)):
with self.subTest(phase=phase, invalid=invalid), self.assertRaises(ValueError):
gate.validate(phase, code, invalid, contract="pending_deadline")
with self.assertRaises(ValueError):
gate.validate(phase, 2, raw, contract="pending_deadline")
red = transcript("RED", "pending_deadline")
for invalid in (red.replace(gate.PENDING_MARKER, "other_assertion"),
red.replace("encrypted_agent_recovery_store_unavailable", "other_error")):
with self.assertRaises(ValueError):
gate.validate("RED", 1, invalid, contract="pending_deadline")

def test_business_named_red_and_green(self):
for phase, code in (("RED", 1), ("GREEN", 0)):
raw = transcript(phase, "business")
gate.validate(phase, code, raw, contract="business")
with self.assertRaises(ValueError):
gate.validate(phase, code, raw)
with self.assertRaises(ValueError):
gate.validate(phase, code, transcript(phase), contract="business")
with self.assertRaises(ValueError):
gate.validate(phase, 2, raw, contract="business")
for noise in ("panic: fixture", "build failed", "setup failed", "no tests to run",
"DATA RACE", "--- SKIP: fixture", "test timed out", "fatal error"):
with self.subTest(phase=phase, noise=noise), self.assertRaises(ValueError):
gate.validate(phase, code, raw, noise, "business")

def test_business_rejects_wrong_assertion_or_test(self):
red = transcript("RED", "business")
for invalid in (
"", red.replace(gate.BUSINESS_MARKER, "unrelated_assertion"),
red.replace('__sub2api_web_search', 'different_tool'),
red.replace('"Action": "run"', '"Action": "skip"', 1),
red.replace(gate.BUSINESS_LEAF, gate.BUSINESS_ROOT + "/wrapped/json"),
red + "\n" + red, red + "\nnot JSON",
red.replace(gate.PACKAGE, "other/package"),
):
with self.subTest(invalid=invalid), self.assertRaises(ValueError):
gate.validate("RED", 1, invalid, contract="business")
with self.assertRaises(ValueError):
gate.validate("RED", 1, red, contract="typo")

def test_named_red_and_green(self):
for phase, code in (("RED", 1), ("GREEN", 0)):
gate.validate(phase, code, transcript(phase))
Expand Down
47 changes: 47 additions & 0 deletions .github/workflows/bps-search.yml
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,53 @@ jobs:
test "$(git rev-parse HEAD)" = "$CANDIDATE_SHA"
git diff --exit-code HEAD
test -z "$(git status --porcelain)"
- name: RED business continuation on pinned current production baseline
shell: bash
run: |
set -euo pipefail
evidence_dir="$RUNNER_TEMP/issue871-evidence"
baseline_sha=da526bfbab47efd40549cab98f1069f9b2b45b9f
fixture_path=backend/internal/service/openai_excel_bps_search_business_test.go
red_dir="$RUNNER_TEMP/issue871-business-red"
git worktree add --detach "$red_dir" "$baseline_sha"
test "$(git -C "$red_dir" rev-parse HEAD)" = "$baseline_sha"
git rev-parse "$baseline_sha" "$baseline_sha^{tree}" > "$evidence_dir/business-baseline.txt"
git ls-tree -r "$baseline_sha" -- backend > "$evidence_dir/business-baseline-blobs.txt"
git show "HEAD:$fixture_path" > "$red_dir/$fixture_path"
git rev-parse "HEAD:$fixture_path" > "$evidence_dir/business-fixture-blob.txt"
test "$(git hash-object "$red_dir/$fixture_path")" = "$(git rev-parse "HEAD:$fixture_path")"
cp "$red_dir/$fixture_path" "$evidence_dir/business_test.go"
git -C "$red_dir" status --short > "$evidence_dir/business-red-status.txt"
test "$(git -C "$red_dir" status --porcelain)" = "?? $fixture_path"
git -C "$red_dir" diff --exit-code HEAD
set +e
(cd "$red_dir/backend" && go test -tags=unit ./internal/service -run '^TestExcelBPSSearchBusinessThroughForward$/^direct$/^json$' -count=1 -json -timeout=5m) > "$evidence_dir/business-red.jsonl" 2> "$evidence_dir/business-red.stderr"
test_status=$?
set -e
printf '%s\n' "$test_status" > "$evidence_dir/business-red-exit.txt"
python3 .github/scripts/issue871-evidence.py RED "$test_status" "$evidence_dir/business-red.jsonl" "$evidence_dir/business-red.stderr" business
test "$(git -C "$red_dir" rev-parse HEAD)" = "$baseline_sha"
git -C "$red_dir" diff --exit-code HEAD
test "$(git -C "$red_dir" status --porcelain)" = "?? $fixture_path"
- name: GREEN business continuation on exact candidate
env:
CANDIDATE_SHA: ${{ github.event.pull_request.head.sha }}
shell: bash
run: |
set -euo pipefail
evidence_dir="$RUNNER_TEMP/issue871-evidence"
test "$(git rev-parse HEAD)" = "$CANDIDATE_SHA"
git diff --exit-code HEAD
test -z "$(git status --porcelain)"
set +e
(cd backend && go test -race -tags=unit ./internal/service -run '^TestExcelBPSSearchBusinessThroughForward$/^direct$/^json$' -count=1 -json -timeout=5m) > "$evidence_dir/business-green.jsonl" 2> "$evidence_dir/business-green.stderr"
test_status=$?
set -e
printf '%s\n' "$test_status" > "$evidence_dir/business-green-exit.txt"
python3 .github/scripts/issue871-evidence.py GREEN "$test_status" "$evidence_dir/business-green.jsonl" "$evidence_dir/business-green.stderr" business
test "$(git rev-parse HEAD)" = "$CANDIDATE_SHA"
git diff --exit-code HEAD
test -z "$(git status --porcelain)"
- name: Repeat real Forward repair ownership and provenance with race detection
working-directory: backend
shell: bash
Expand Down
5 changes: 4 additions & 1 deletion .github/workflows/ws-readable-continuation.yml
Original file line number Diff line number Diff line change
Expand Up @@ -218,9 +218,12 @@ jobs:
# PR855 route-evidence types and later production BPS search APIs
# are absent in this baseline. Their unselected tests follow the
# pinned historical APIs too; the selected ticket oracle stays current.
# Later UltraFast fields and the search-business helper dependency
# are absent too. Restore only these unrelated fixtures; never the
# selected ticket oracle or any candidate GREEN fixture.
# BPS search is already excluded with blob/contract proofs by the
# shared helpers above, so do not inspect/remove it a second time.
for target in backend/internal/service/temp_unsched_pending_test.go backend/internal/service/temp_unsched_stream_timing_test.go backend/internal/service/temp_unsched_ws_forward_test.go backend/internal/service/openai_auth_connection_status_test.go backend/internal/service/openai_auth_connections_account_type_test.go backend/internal/service/account_ops_balance_test.go backend/internal/service/openai_excel_bps_bridge_failure_test.go backend/internal/service/quality_5xx_immediate_test.go backend/internal/service/openai_ws_agent_recovery_test.go; do
for target in backend/internal/service/temp_unsched_pending_test.go backend/internal/service/temp_unsched_stream_timing_test.go backend/internal/service/temp_unsched_ws_forward_test.go backend/internal/service/openai_auth_connection_status_test.go backend/internal/service/openai_auth_connections_account_type_test.go backend/internal/service/account_ops_balance_test.go backend/internal/service/openai_excel_bps_bridge_failure_test.go backend/internal/service/quality_5xx_immediate_test.go backend/internal/service/openai_ws_agent_recovery_test.go backend/internal/service/admin_group_duplicate_test.go backend/internal/service/admin_service_group_test.go backend/internal/service/api_key_auth_cache_openai_fast_test.go backend/internal/service/free_ultrafast_customer_cost_test.go backend/internal/service/openai_excel_bps_search_business_test.go; do
candidate_blob=$(git rev-parse "$CANDIDATE_SHA:$target")
test "$(git hash-object "$tested_tree/$target")" = "$candidate_blob"
if git cat-file -e "$baseline:$target" 2>/dev/null; then
Expand Down
Loading
Loading