Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
73 changes: 73 additions & 0 deletions .github/scripts/prepare-ws-inherited-model-red.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
"""Materialize the entire pinned baseline plus one identical candidate test.

No historical overlay helper or existing worktree is modified. All archived
backend blobs (including dependencies and historical fixtures) are verified.
"""

import json
import shutil
import subprocess
import sys
from pathlib import Path

BASELINE = "da526bfbab47efd40549cab98f1069f9b2b45b9f"
FIXTURE = "backend/internal/handler/openai_ws_inherited_model_test.go"
GUARDS = "backend/internal/service/openai_ws_owner_conflict_test.go"


def git(*args, **kwargs):
return subprocess.check_output(["git", *args], **kwargs)


def prepare(destination, evidence):
destination, evidence = Path(destination).resolve(), Path(evidence).resolve()
candidate = git("rev-parse", "HEAD").decode().strip()
# A fresh directory avoids deleting or changing another writer's files.
destination.mkdir(parents=True, exist_ok=False)
evidence.mkdir(parents=True, exist_ok=True)
archive = subprocess.Popen(["git", "archive", BASELINE], stdout=subprocess.PIPE)
try:
subprocess.run(["tar", "-x", "-C", str(destination)], stdin=archive.stdout, check=True)
finally:
archive.stdout.close()
if archive.wait() != 0:
raise RuntimeError("baseline archive failed")
entries = []
for row in git("ls-tree", "-rz", BASELINE, "--", "backend").split(b"\0"):
if not row:
continue
metadata, path = row.decode().split("\t", 1)
mode, kind, blob = metadata.split()
if kind != "blob" or mode not in {"100644", "100755"}:
raise ValueError(f"unhandled baseline entry: {path}")
entries.append((path, blob))
paths = "".join(str(destination / path) + "\n" for path, _ in entries)
actual = git("hash-object", "--stdin-paths", input=paths.encode()).decode().splitlines()
if actual != [blob for _, blob in entries]:
raise ValueError("baseline backend blob mismatch")
if (destination / FIXTURE).exists():
raise ValueError("fixture unexpectedly exists in pinned baseline")
fixture_blob = git("rev-parse", f"{candidate}:{FIXTURE}").decode().strip()
if git("hash-object", FIXTURE).decode().strip() != fixture_blob:
raise ValueError("dirty candidate fixture")
if Path(GUARDS).read_bytes() != (destination / GUARDS).read_bytes():
raise ValueError("guard fixture changed; identical baseline/candidate guard tests required")
shutil.copyfile(FIXTURE, destination / FIXTURE)
if git("hash-object", str(destination / FIXTURE)).decode().strip() != fixture_blob:
raise ValueError("candidate test copy mismatch")
manifest = {
"baseline": BASELINE, "candidate": candidate,
"baseline_tree": git("rev-parse", f"{BASELINE}^{{tree}}").decode().strip(),
"baseline_backend_tree": git("rev-parse", f"{BASELINE}:backend").decode().strip(),
"candidate_fixture_blob": fixture_blob, "fixture": FIXTURE,
"guard_fixture_blob": git("hash-object", GUARDS).decode().strip(),
"verified_baseline_backend_blobs": len(entries),
"production_changes_in_red": [], "candidate_test_overlays": [FIXTURE],
}
(evidence / "source-manifest.json").write_text(json.dumps(manifest, indent=2) + "\n")
(evidence / "baseline-backend-blobs.txt").write_text("".join(f"{blob} {path}\n" for path, blob in entries))
print(json.dumps(manifest, sort_keys=True))


if __name__ == "__main__":
prepare(*sys.argv[1:])
134 changes: 134 additions & 0 deletions .github/scripts/test_ws_inherited_model_evidence.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,134 @@
"""Synthetic parser checks only; these are never Go RED/GREEN evidence."""

import importlib.util
import json
import unittest
from pathlib import Path

SPEC = importlib.util.spec_from_file_location(
"evidence", Path(__file__).with_name("ws-inherited-model-evidence.py")
)
GATE = importlib.util.module_from_spec(SPEC)
SPEC.loader.exec_module(GATE)


def transcript(suite="handler", phase="RED"):
package, tests, red, positives, fences = GATE.contract(suite)
package = "github.com/Wei-Shaw/sub2api/internal/" + package
failures = set()
if red and phase == "RED":
for leaf in red:
parts = leaf.split("/")
failures.update("/".join(parts[:i]) for i in range(1, len(parts) + 1))
failures.add(None)
rows = [{"Package": package, "Action": "start"}]
for name in sorted(tests):
rows.append({"Package": package, "Action": "run", "Test": name})
if suite == "pending" and name in positives:
output = "PENDING_OBSERVATION_LOST: owner_pending=1" if name in failures else "PENDING_OBSERVATION_CONTROL_OK: classification preserved; no ownership publication"
elif name in red and name in failures:
output = GATE.MARKER + " T1/T2 completed on A; T2 omitted model; O1/O2 and C delivered; proof=failed; T3 tool_provenance_unknown; A disabled; B sends=0; records unchanged"
elif name in positives:
output = "INHERITED_MODEL_TOOL_LOOP_OK: B sends=1; executions=1; A disabled"
elif name in fences:
output = "INHERITED_MODEL_FENCE_OK: B sends=0; records unchanged"
else:
output = "synthetic parent event"
rows.append({"Package": package, "Action": "output", "Test": name, "Output": output})
rows.append({"Package": package, "Action": "fail" if name in failures else "pass", "Test": name})
rows.append({"Package": package, "Action": "fail" if failures else "pass"})
return rows


def encode(rows):
return "\n".join(json.dumps(row) for row in rows) + "\n"


class EvidenceTests(unittest.TestCase):
def test_complete_contracts(self):
for suite in ("handler", "guards", "guards_with_client_model", "pending"):
for phase in ("RED", "GREEN"):
with self.subTest(suite=suite, phase=phase):
status = int(suite in {"handler", "pending"} and phase == "RED")
result = GATE.validate(suite, phase, status, encode(transcript(suite, phase)))
self.assertEqual(status * len(GATE.contract(suite)[2]), result["behavioral_red_leaves"])

def test_pending_requires_exact_failure_and_all_error_controls(self):
for change in ("marker", "missing_backend_error", "failed_control", "skip", "duplicate", "wrong_status"):
with self.subTest(change=change):
rows = transcript("pending")
status = 1
if change == "marker":
for row in rows:
row["Output"] = row.get("Output", "").replace("PENDING_OBSERVATION_LOST:", "other failure")
elif change == "missing_backend_error":
rows = [r for r in rows if r.get("Test") != GATE.PENDING_ROOT + "/pending_backend_deadline"]
elif change == "failed_control":
for row in rows:
if row.get("Test") == GATE.PENDING_ROOT + "/pending_backend_deadline" and row["Action"] == "pass":
row["Action"] = "fail"
elif change == "skip":
rows[3]["Action"] = "skip"
elif change == "duplicate":
rows.append(rows[-1])
else:
status = 0
with self.assertRaises(ValueError):
GATE.validate("pending", "RED", status, encode(rows))

def test_runtime_build_and_skip_contamination(self):
for diagnostic in ("[build failed]", "undefined: fixture", "panic: fixture", "WARNING: DATA RACE", "test timed out after 5m", "no tests to run", "--- SKIP: fixture", "signal: killed"):
for decoded in (False, True):
with self.subTest(diagnostic=diagnostic, decoded=decoded):
rows = transcript()
if decoded:
rows[2]["Output"] += "\n" + diagnostic
raw = encode(rows)
else:
raw = encode(rows) + diagnostic
with self.assertRaises((ValueError, json.JSONDecodeError)):
GATE.validate("handler", "RED", 1, raw)

def test_missing_repeated_unrelated_and_wrong_outcomes(self):
for mutation in ("missing_control", "duplicate_run", "duplicate_terminal", "unrelated", "package", "skip", "wrong_failure", "missing_marker", "truncated", "empty"):
with self.subTest(mutation=mutation):
rows = transcript()
if mutation == "missing_control":
rows = [r for r in rows if r.get("Test") != GATE.FENCES + "/function/pending"]
elif mutation == "duplicate_run":
rows.append(dict(rows[1]))
elif mutation == "duplicate_terminal":
rows.append(dict(rows[-1]))
elif mutation == "unrelated":
rows.append({"Package": rows[0]["Package"], "Action": "fail", "Test": "TestOther"})
elif mutation == "package":
rows[0]["Package"] += "_other"
elif mutation == "skip":
rows[3]["Action"] = "skip"
elif mutation == "wrong_failure":
for row in rows:
if row.get("Action") == "pass" and row.get("Test") == GATE.FENCES + "/function/pending":
row["Action"] = "fail"
elif mutation == "missing_marker":
for row in rows:
row["Output"] = row.get("Output", "").replace(GATE.MARKER, "unrelated assertion")
elif mutation == "truncated":
rows = rows[:-1]
elif mutation == "empty":
rows = []
with self.assertRaises(ValueError):
GATE.validate("handler", "RED", 1, encode(rows))

def test_status_phase_and_malformed_input(self):
for status in (0, 2, 137):
with self.subTest(status=status), self.assertRaises(ValueError):
GATE.validate("handler", "RED", status, encode(transcript()))
with self.assertRaises(ValueError):
GATE.validate("handler", "GREEN", 0, encode(transcript()))
for raw in ("{malformed\n", "[]\n", "null\n", "not JSON\n"):
with self.subTest(raw=raw), self.assertRaises(ValueError):
GATE.validate("handler", "RED", 1, raw)


if __name__ == "__main__":
unittest.main()
121 changes: 121 additions & 0 deletions .github/scripts/ws-inherited-model-evidence.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,121 @@
"""Accept only the complete authenticated inherited-model behavioral matrix."""

import importlib.util
import json
import sys
from collections import Counter
from pathlib import Path

SPEC = importlib.util.spec_from_file_location(
"clean", Path(__file__).with_name("assert-readable-continuation.py")
)
CLEAN = importlib.util.module_from_spec(SPEC)
SPEC.loader.exec_module(CLEAN)

POSITIVE = "TestOpenAIWSInheritedModelToolProof"
FENCES = "TestOpenAIWSInheritedModelProofFences"
SCENARIOS = ("explicit", "inherited", "mapped_explicit", "mapped_inherited")
NEGATIVES = (
"unknown", "pending", "epoch", "arguments", "terminal_excludes",
"model_change", "same_wire_model_change",
)
KINDS = ("function", "custom")
MARKER = "INHERITED_MODEL_TOOL_PROOF_NOT_RESUMED:"
PENDING_ROOT = "TestOpenAIAgentPendingObservation"
PENDING_CASES = (
"pending_get_deadline", "pending_get_cancel", "latest_pending_get_deadline", "pending_timer_deadline",
"initial_backend_error", "initial_backend_deadline", "pending_backend_error", "pending_backend_deadline",
"completed", "failed", "conflicted", "missing",
)


def contract(suite):
if suite == "pending":
leaves = {f"{PENDING_ROOT}/{case}" for case in PENDING_CASES}
red = {f"{PENDING_ROOT}/{case}" for case in ("pending_get_deadline", "pending_get_cancel", "latest_pending_get_deadline")}
return "service", leaves | {PENDING_ROOT}, red, leaves, set()
if suite == "handler":
roots = {POSITIVE, FENCES}
parents = {f"{root}/{kind}" for root in roots for kind in KINDS}
good = {f"{POSITIVE}/{kind}/{case}" for kind in KINDS for case in SCENARIOS}
fences = {f"{FENCES}/{kind}/{case}" for kind in KINDS for case in NEGATIVES}
red = {name for name in good if name.endswith("/inherited") or name.endswith("/mapped_inherited")}
return "handler", roots | parents | good | fences, red, good, fences
if suite in {"guards", "guards_with_client_model"}:
cases = {
"TestOpenAIToolProvenanceDeliveryAndScope": (
"completed", "early_only", "write_failed", "terminal_excludes", "duplicate",
),
"TestOpenAIWSMixedOwnerFinalSendFences": (
"none", "pending", "failed", "conflicted", "missing", "changed", "input", "tools", "call_proof",
),
}
if suite == "guards_with_client_model":
cases["TestOpenAIWSRecoveryEffectiveClientModel"] = (
"no_ws_preflight", "inherited", "explicit_switch", "inherited_after_switch",
"explicit_overrides_preflight", "separate_connection",
)
tests = set(cases) | {f"{root}/{case}" for root, leaves in cases.items() for case in leaves}
return "service", tests, set(), set(), set()
raise ValueError("unknown suite")


def validate(suite, phase, status, transcript):
if phase not in {"RED", "GREEN"}:
raise ValueError("unknown phase")
CLEAN.check_clean(transcript)
package, tests, red_leaves, positives, fences = contract(suite)
package = "github.com/Wei-Shaw/sub2api/internal/" + package
rows = []
for line in transcript.splitlines():
# Go can print dependency downloads on stderr before its JSON stream.
if line.startswith("go: downloading "):
continue
row = json.loads(line)
if not isinstance(row, dict) or row.get("Package") != package:
raise ValueError("unexpected package or malformed event")
if row.get("Action") not in {"start", "run", "pause", "cont", "output", "pass", "fail"}:
raise ValueError("unexpected action (including skip/build failure)")
if row.get("Test") not in tests | {None}:
raise ValueError("unrelated test")
rows.append(row)
counts = Counter((r.get("Action"), r.get("Test")) for r in rows)
failures = set()
if red_leaves and phase == "RED":
for leaf in red_leaves:
parts = leaf.split("/")
failures.update("/".join(parts[:i]) for i in range(1, len(parts) + 1))
failures.add(None)
if status != (1 if failures else 0):
raise ValueError("wrong process exit status")
if counts["start", None] != 1 or {r.get("Test") for r in rows if r.get("Action") == "run"} != tests:
raise ValueError("missing/repeated package or test run")
for name in tests | {None}:
if name is not None and counts["run", name] != 1:
raise ValueError(f"missing/repeated run: {name}")
expected = "fail" if name in failures else "pass"
if counts[expected, name] != 1 or counts["pass", name] + counts["fail", name] != 1:
raise ValueError(f"wrong/missing/repeated outcome: {name}")
for name in positives | fences:
output = "".join(r.get("Output", "") for r in rows if r.get("Test") == name)
if suite == "pending":
required = ("PENDING_OBSERVATION_LOST:", "owner_pending=1") if name in failures else (
"PENDING_OBSERVATION_CONTROL_OK:", "classification preserved", "no ownership publication",
)
elif name in failures:
required = (
MARKER, "T1/T2 completed on A", "T2 omitted model", "O1/O2 and C delivered",
"proof=failed", "T3 tool_provenance_unknown", "A disabled", "B sends=0", "records unchanged",
)
elif name in fences:
required = ("INHERITED_MODEL_FENCE_OK:", "B sends=0", "records unchanged")
else:
required = ("INHERITED_MODEL_TOOL_LOOP_OK:", "B sends=1", "executions=1", "A disabled")
if any(text not in output for text in required):
raise ValueError(f"missing behavioral evidence: {name}")
return {"suite": suite, "phase": phase, "exit_status": status, "tests": len(tests), "behavioral_red_leaves": len(red_leaves) if failures else 0}


if __name__ == "__main__":
suite, phase, status, path = sys.argv[1:]
print(json.dumps(validate(suite, phase, int(status), Path(path).read_text()), sort_keys=True))
Loading
Loading