diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1be0c564..450edcfe 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -46,6 +46,7 @@ jobs: python3 -m pip install pytest ruff==0.15.22 python3 -m pip install . --no-deps python3 -m pip install 'quant-strategy-plugins[ai] @ git+https://github.com/QuantStrategyLab/QuantStrategyPlugins.git@6b76d512c8273deb804c0770a203558284778399' + python3 -m pip install 'quant-platform-kit @ git+https://github.com/QuantStrategyLab/QuantPlatformKit.git@7363011d56926d39f4fffeb036e511391114e39f' python3 -m pip check python3 -m ruff check . python3 -m pytest tests ops/quant-monitor/tests -q diff --git a/.github/workflows/research_input_readback.yml b/.github/workflows/research_input_readback.yml index 323570ae..45ca611d 100644 --- a/.github/workflows/research_input_readback.yml +++ b/.github/workflows/research_input_readback.yml @@ -11,6 +11,7 @@ on: options: - readback - soxl_learning + - soxl_validation blend_gate_mid_soxl_weights: description: Reviewed baseline and up to two bounded learning values required: true @@ -26,27 +27,36 @@ concurrency: jobs: readback: - if: github.repository == 'QuantStrategyLab/AIAuditBridge' && github.ref == 'refs/heads/main' && (inputs.operation != 'soxl_learning' || github.run_attempt == 1) + if: github.repository == 'QuantStrategyLab/AIAuditBridge' && github.ref == 'refs/heads/main' && (inputs.operation == 'readback' || github.run_attempt == 1) runs-on: [self-hosted, codex-vps] - timeout-minutes: ${{ inputs.operation == 'soxl_learning' && 35 || 10 }} + timeout-minutes: ${{ inputs.operation != 'readback' && 35 || 10 }} permissions: contents: read id-token: write + actions: read steps: - name: Checkout uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: persist-credentials: false - - name: Initialize the sanitized learning record - if: inputs.operation == 'soxl_learning' + - name: Set bounded validation temporary paths + env: + VALIDATION_CONTROL_ENV: ${{ runner.temp }}/aab-soxl-validation-control-${{ github.run_id }}-${{ github.run_attempt }} + DEVELOPMENT_SOURCE_ROOT: ${{ runner.temp }}/aab-soxl-validation-source-${{ github.run_id }}-${{ github.run_attempt }} + run: | + printf 'VALIDATION_CONTROL_ENV=%s\nDEVELOPMENT_SOURCE_ROOT=%s\n' "$VALIDATION_CONTROL_ENV" "$DEVELOPMENT_SOURCE_ROOT" >> "$GITHUB_ENV" + + - name: Initialize the sanitized research record + if: inputs.operation != 'readback' env: LEARNING_OUTPUT: ${{ runner.temp }}/aab-soxl-learning-${{ github.run_id }}-${{ github.run_attempt }}/summary.json + OPERATION: ${{ inputs.operation }} run: | set -euo pipefail umask 077 mkdir -m 700 "$(dirname "$LEARNING_OUTPUT")" - python3 -c 'import os; from pathlib import Path; from scripts.run_soxl_manual_learning import initialize_record; initialize_record(Path(os.environ["LEARNING_OUTPUT"]), {"repository": os.environ["GITHUB_REPOSITORY"], "ref": os.environ["GITHUB_REF"], "event_name": os.environ["GITHUB_EVENT_NAME"], "actor": os.environ["GITHUB_ACTOR"], "run_id": os.environ["GITHUB_RUN_ID"], "run_attempt": os.environ["GITHUB_RUN_ATTEMPT"]})' + python3 -c 'import os; from pathlib import Path; from scripts.run_soxl_manual_learning import initialize_record; initialize_record(Path(os.environ["LEARNING_OUTPUT"]), {"repository": os.environ["GITHUB_REPOSITORY"], "ref": os.environ["GITHUB_REF"], "event_name": os.environ["GITHUB_EVENT_NAME"], "actor": os.environ["GITHUB_ACTOR"], "run_id": os.environ["GITHUB_RUN_ID"], "run_attempt": os.environ["GITHUB_RUN_ATTEMPT"]}, operation=os.environ["OPERATION"])' - name: Checkout the frozen original P1 validator uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 @@ -57,16 +67,16 @@ jobs: persist-credentials: false - name: Checkout the bounded learning consumer - if: inputs.operation == 'soxl_learning' + if: inputs.operation != 'readback' uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: repository: QuantStrategyLab/UsEquitySnapshotPipelines - ref: b03ecbe4e0a7a0de22f298499f867a7039e4b60a + ref: ${{ inputs.operation == 'soxl_validation' && 'b68b4a81ccdab7b61042fcf98df0189dfe539ef4' || 'b03ecbe4e0a7a0de22f298499f867a7039e4b60a' }} path: consumer-source persist-credentials: false - name: Checkout the frozen strategy runtime - if: inputs.operation == 'soxl_learning' + if: inputs.operation != 'readback' uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: repository: QuantStrategyLab/UsEquityStrategies @@ -87,12 +97,12 @@ jobs: env -u UV_PYTHON -u VIRTUAL_ENV uv sync --locked --no-dev --no-editable --python 3.11 - name: Install the bounded learning consumer runtime - if: inputs.operation == 'soxl_learning' + if: inputs.operation != 'readback' working-directory: consumer-source run: env -u UV_PYTHON -u VIRTUAL_ENV uv sync --locked --no-dev --no-editable --python 3.11 - name: Install the frozen strategy runtime - if: inputs.operation == 'soxl_learning' + if: inputs.operation != 'readback' working-directory: ues-source env: UV_PROJECT_ENVIRONMENT: ${{ runner.temp }}/aab-soxl-ues-env-${{ github.run_id }}-${{ github.run_attempt }} @@ -101,6 +111,37 @@ jobs: rm -rf -- build test -z "$(git status --porcelain --untracked-files=all)" + - name: Checkout the published strict gate + if: inputs.operation == 'soxl_validation' + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + repository: QuantStrategyLab/QuantPlatformKit + ref: 7363011d56926d39f4fffeb036e511391114e39f + path: validation-gate-source + persist-credentials: false + + - name: Install the isolated strict gate + if: inputs.operation == 'soxl_validation' + run: | + set -euo pipefail + python3 -m venv "$VALIDATION_CONTROL_ENV" + "$VALIDATION_CONTROL_ENV/bin/python" -m pip install --quiet --no-deps ./validation-gate-source + "$VALIDATION_CONTROL_ENV/bin/python" -c 'from quant_platform_kit.strategy_lifecycle.research_promotion_cycle import enforce_promotion_backtest_gates; from scripts.run_soxl_manual_learning import run_manual_validation' + + - name: Read the original completed development summary + if: inputs.operation == 'soxl_validation' + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + umask 077 + mkdir -m 700 "$DEVELOPMENT_SOURCE_ROOT" + if ! gh run download 34376283866 --repo QuantStrategyLab/AIAuditBridge --name soxl-manual-learning-34376283866-1 --dir "$DEVELOPMENT_SOURCE_ROOT" > "$DEVELOPMENT_SOURCE_ROOT/download.log" 2>&1; then + echo '{"status":"parked","failure_stage":"development_summary_unavailable"}' + exit 1 + fi + DEVELOPMENT_SUMMARY="$DEVELOPMENT_SOURCE_ROOT/summary.json" "$VALIDATION_CONTROL_ENV/bin/python" -c 'import json, os; from pathlib import Path; from scripts.run_soxl_manual_learning import _summary_digest, DEVELOPMENT_SUMMARY_SHA256; p=Path(os.environ["DEVELOPMENT_SUMMARY"]); assert not p.is_symlink() and p.stat().st_size <= 1048576; assert _summary_digest(json.loads(p.read_text())) == DEVELOPMENT_SUMMARY_SHA256' + # Direct federation grants only object reads on the fixed P1 root below. # The provider additionally binds numeric repo/owner IDs, main and this workflow. # No service-account key, user ADC, daemon credential or AI invocation is used. @@ -262,10 +303,25 @@ jobs: --actor "$GITHUB_ACTOR" \ --run-id "$GITHUB_RUN_ID" \ --run-attempt "$GITHUB_RUN_ATTEMPT" + elif [ "$OPERATION" = "soxl_validation" ]; then + cd "$GITHUB_WORKSPACE" + UV_PROJECT_ENVIRONMENT="$UES_ENV_ROOT" "$VALIDATION_CONTROL_ENV/bin/python" -m scripts.run_soxl_manual_learning \ + --development-summary "$DEVELOPMENT_SOURCE_ROOT/summary.json" \ + --manifest-sha256 "$EXPECTED_MANIFEST_SHA256" \ + --root "$READBACK_ROOT/root" \ + --consumer-source "$CONSUMER_SOURCE" \ + --ues-source "$UES_SOURCE" \ + --output "$LEARNING_OUTPUT" \ + --repository "$GITHUB_REPOSITORY" \ + --ref "$GITHUB_REF" \ + --event-name "$GITHUB_EVENT_NAME" \ + --actor "$GITHUB_ACTOR" \ + --run-id "$GITHUB_RUN_ID" \ + --run-attempt "$GITHUB_RUN_ATTEMPT" fi - - name: Upload the sanitized learning record - if: always() && inputs.operation == 'soxl_learning' + - name: Upload the sanitized research record + if: always() && inputs.operation != 'readback' uses: actions/upload-artifact@v7 with: name: soxl-manual-learning-${{ github.run_id }}-${{ github.run_attempt }} @@ -279,4 +335,4 @@ jobs: READBACK_ROOT: ${{ runner.temp }}/aab-research-input-${{ github.run_id }}-${{ github.run_attempt }} LEARNING_OUTPUT_ROOT: ${{ runner.temp }}/aab-soxl-learning-${{ github.run_id }}-${{ github.run_attempt }} UES_ENV_ROOT: ${{ runner.temp }}/aab-soxl-ues-env-${{ github.run_id }}-${{ github.run_attempt }} - run: rm -rf -- "$READBACK_ROOT" "$LEARNING_OUTPUT_ROOT" "$UES_ENV_ROOT" + run: rm -rf -- "$READBACK_ROOT" "$LEARNING_OUTPUT_ROOT" "$UES_ENV_ROOT" "$VALIDATION_CONTROL_ENV" "$DEVELOPMENT_SOURCE_ROOT" diff --git a/scripts/run_soxl_manual_learning.py b/scripts/run_soxl_manual_learning.py index 4c4a7f41..76690ae2 100644 --- a/scripts/run_soxl_manual_learning.py +++ b/scripts/run_soxl_manual_learning.py @@ -4,6 +4,7 @@ from __future__ import annotations import argparse +import hashlib import json import math import re @@ -41,6 +42,17 @@ EXPECTED_EVENT = "workflow_dispatch" SAFE_REASON = re.compile(r"[a-z0-9_]{1,64}\Z") WATCHER_MARKER_PREFIX = "qsl-soxl-watcher-learning:v1" +DEVELOPMENT_SUMMARY_SHA256 = "89418d4e13efa9379f91c522ccbe084e2cbf180ba343103d5b73fb7cdbb955a8" +VALIDATION_QPK_REVISION = "7363011d56926d39f4fffeb036e511391114e39f" +VALIDATION_CONSUMER_REVISION = "b68b4a81ccdab7b61042fcf98df0189dfe539ef4" +VALIDATION_FOLDS = [ + dict(zip(("train_start", "train_end", "test_start", "test_end"), boundaries, strict=True)) + for boundaries in ( + ("2022-12-28", "2023-06-30", "2023-07-03", "2023-12-29"), + ("2024-01-02", "2024-06-28", "2024-07-01", "2024-12-31"), + ("2025-01-02", "2025-02-28", "2025-03-03", "2025-07-31"), + ) +] class ManualLearningError(ValueError): @@ -724,19 +736,192 @@ def run_manual_learning( return artifact +def _summary_digest(value: object) -> str: + return hashlib.sha256(json.dumps(value, sort_keys=True, separators=(",", ":"), allow_nan=False).encode()).hexdigest() + + +def _strict_validation_summary(value: object, gate: Callable[..., tuple[bool, str]]) -> dict[str, Any]: + from dataclasses import fields + from quant_platform_kit.strategy_lifecycle.contracts import BacktestValidationIdentity, OptimizationProposal + + profile = "soxl_soxx_three_asset_mid_weight_learning_v1" + if ( + not isinstance(value, Mapping) or value.get("status") != "PROMOTION_BACKTEST_RUNS_BUILT" + or value.get("stage") != "promotion_validation" + or any(value.get(key) is not True for key in ("learning_only", "no_order", "size_zero_required")) + or value.get("promotion_eligible") is not False + or value.get("live_authority_granted") is not False + ): + raise ManualLearningError("validation_result_invalid") + raw_proposal = value.get("proposal") + method = f"bounded_development_tradeoff:sha256:{DEVELOPMENT_SUMMARY_SHA256}" + if ( + not isinstance(raw_proposal, Mapping) + or raw_proposal.get("strategy_profile") != profile or raw_proposal.get("domain") != "us_equity" + or raw_proposal.get("current_params") != {"blend_gate_mid_soxl_weight": 0.65} + or raw_proposal.get("proposed_params") != {"blend_gate_mid_soxl_weight": 0.55} + or raw_proposal.get("optimization_method") != method + or raw_proposal.get("recommendation") != "research_candidate" + or raw_proposal.get("walk_forward_passed") is not False + ): + raise ManualLearningError("validation_proposal_invalid") + proposal = OptimizationProposal( + strategy_profile=profile, domain="us_equity", current_params=raw_proposal["current_params"], + proposed_params=raw_proposal["proposed_params"], optimization_method=method, + recommendation="research_candidate", search_iterations=3, + ) + safe_runs: dict[str, list[dict[str, Any]]] = {} + metric_keys = ("cagr", "sharpe_ratio", "max_drawdown", "volatility", "total_return") + result_keys = (*metric_keys, "strategy_profile", "domain", "param_set_id", "params", "start_date", "end_date", "observation_count", "source_revision", "cost_model", "cost_inputs") + for role, weight in (("baseline", 0.65), ("candidate", 0.55)): + runs = value.get(f"{role}_promotion_runs") + if not isinstance(runs, list) or len(runs) != len(COST_BPS): + raise ManualLearningError("validation_result_invalid") + safe_runs[role] = [] + for run, cost in zip(runs, COST_BPS, strict=True): + if not isinstance(run, Mapping): + raise ManualLearningError("validation_result_invalid") + passed, _ = gate(proposal, { + "status": "PASS", "orchestrator": "BacktestOrchestrator", "protocol": "purged_walk_forward.v1", + "locked_independent_oos": {"locked": True, "independent": True, "reused_for_selection": False}, + "promotion_run": run, + }) + if not passed: + raise ManualLearningError("strict_backtest_gate_rejected") + if ( + run.get("source_revision") != UES_REVISION + or run.get("locked_oos_start") != "2025-08-04" or run.get("locked_oos_end") != "2026-08-04" + or run.get("folds") != VALIDATION_FOLDS + or run.get("purge_days") != 1 or run.get("embargo_days") != 1 + or run.get("cost_model") != {"model_id": f"all_in_per_side_{cost:g}bps", "commission_bps": 0.0, "slippage_bps": cost, "market_impact_bps": 0.0} + or not isinstance(run.get("fold_results"), list) or len(run["fold_results"]) != 3 + ): + raise ManualLearningError("validation_result_invalid") + safe_results = [] + windows = [(fold["test_start"], fold["test_end"]) for fold in run["folds"]] + windows.append((run["locked_oos_start"], run["locked_oos_end"])) + for index, (result, (start, end)) in enumerate(zip([*run["fold_results"], run.get("locked_oos_result")], windows, strict=True)): + suffix = f"_wf{index}" if index < 3 else "_locked_oos" + result_id = f"soxl-three-asset-{DEVELOPMENT_SUMMARY_SHA256}-{role}-cost-{cost:g}{suffix}" + if ( + not isinstance(result, Mapping) or result.get("strategy_profile") != profile + or result.get("domain") != "us_equity" + or result.get("params") != {"blend_gate_mid_soxl_weight": weight} + or result.get("source_revision") != UES_REVISION + or result.get("param_set_id") != result_id + or result.get("cost_model") != run["cost_model"]["model_id"] + or result.get("cost_inputs") != {key: run["cost_model"][key] for key in ("commission_bps", "slippage_bps", "market_impact_bps")} + or not isinstance(result.get("observation_count"), int) or isinstance(result["observation_count"], bool) + or result["observation_count"] < 2 + or result.get("start_date") != start or result.get("end_date") != end + or any(isinstance(result.get(key), bool) or not isinstance(result.get(key), (int, float)) or not math.isfinite(result[key]) for key in metric_keys) + ): + raise ManualLearningError("validation_result_invalid") + identity = result.get("validation_identity") + expected_identity = { + "protocol": "purged_walk_forward.v1", "fold_id": result_id, + "fold_role": "test" if index < 3 else "locked_oos", + "train_start": VALIDATION_FOLDS[index]["train_start"] if index < 3 else None, + "train_end": VALIDATION_FOLDS[index]["train_end"] if index < 3 else None, + "test_start": start, "test_end": end, + "locked_oos_start": "2025-08-04", "locked_oos_end": "2026-08-04", + "purge_days": 1, "embargo_days": 1, + } + if identity != expected_identity: + raise ManualLearningError("validation_result_invalid") + safe_results.append({ + **{key: result.get(key) for key in result_keys}, + "validation_identity": {field.name: identity.get(field.name) for field in fields(BacktestValidationIdentity)}, + }) + safe_runs[role].append({ + **{key: run[key] for key in ("strategy_profile", "domain", "folds", "locked_oos_start", "locked_oos_end", "purge_days", "embargo_days", "source_revision", "cost_model")}, + "fold_results": safe_results[:3], "locked_oos_result": safe_results[3], + }) + comparisons = [] + for baseline, candidate, cost in zip(safe_runs["baseline"], safe_runs["candidate"], COST_BPS, strict=True): + b, c = baseline["locked_oos_result"], candidate["locked_oos_result"] + comparisons.append({ + "cost_bps": cost, "baseline_max_drawdown": b["max_drawdown"], "candidate_max_drawdown": c["max_drawdown"], + "baseline_cagr": b["cagr"], "candidate_cagr": c["cagr"], + "baseline_sharpe": b["sharpe_ratio"], "candidate_sharpe": c["sharpe_ratio"], + "cagr_retention": c["cagr"] / b["cagr"] if b["cagr"] > 0 else None, + "sharpe_retention": c["sharpe_ratio"] / b["sharpe_ratio"] if b["sharpe_ratio"] > 0 else None, + }) + return { + "proposal": proposal.to_dict(), "baseline_promotion_runs": safe_runs["baseline"], + "candidate_promotion_runs": safe_runs["candidate"], "oos_comparison": comparisons, + "strict_backtest_gate": {"status": "passed", "checks": 6, "qpk_revision": VALIDATION_QPK_REVISION}, + "human_quality_decision_required": True, + } + + +def run_manual_validation( + *, development_summary: Path, manifest_sha256: str, root: Path, consumer_source: Path, + ues_source: Path, context: Mapping[str, str], command_runner: Callable[[list[str]], Any] | None = None, + progress_writer: Callable[[Mapping[str, Any]], None] | None = None, +) -> dict[str, Any]: + _validate_authority(context) + try: + if development_summary.is_symlink() or development_summary.stat().st_size > 1_048_576: + raise ValueError + if _summary_digest(json.loads(development_summary.read_text())) != DEVELOPMENT_SUMMARY_SHA256: + raise ValueError + except (OSError, ValueError, TypeError): + raise ManualLearningError("validation_development_source_invalid") from None + try: + from quant_platform_kit.strategy_lifecycle.research_promotion_cycle import enforce_promotion_backtest_gates + except ImportError: + raise ManualLearningError("strict_validation_runtime_unavailable") from None + required = (root / "binding.json", root / "manifest.json", root / "bars.json", consumer_source / "scripts/run_soxl_three_asset_learning.py", consumer_source / "config/soxl_soxx_core_only_p2_v3.json") + if ( + re.fullmatch(r"[0-9a-f]{64}", manifest_sha256) is None + or any(path.is_symlink() or not path.is_file() for path in required) + or not (consumer_source / ".venv/bin/python").is_file() + or ues_source.is_symlink() or not ues_source.is_dir() + ): + raise ManualLearningError("source_or_input_unavailable") + artifact = _safe_base(context, (0.65, 0.55), manifest_sha256) + artifact.update(operation="soxl_validation", status="parked", development_summary_sha256=DEVELOPMENT_SUMMARY_SHA256) + artifact["consumer_source"]["revision"] = VALIDATION_CONSUMER_REVISION + artifact.update(numeric_execution={"status": "started"}, research_executed=None) + if progress_writer is not None: + progress_writer(dict(artifact)) + command = _numeric_command(root, consumer_source, ues_source, ()) + command.extend(("--promotion-validation-development-summary", str(development_summary))) + runner = command_runner or (lambda argv: subprocess.run(argv, capture_output=True, text=True, timeout=1200, check=False)) + try: + completed = runner(command) + except (OSError, subprocess.SubprocessError): + artifact.update(failure_stage="numeric_outcome_unknown", numeric_execution={"status": "outcome_unknown"}) + return artifact + if getattr(completed, "returncode", None) != 0: + artifact.update(failure_stage="numeric_execution_failed", numeric_execution={"status": "failed"}) + return artifact + try: + safe = _strict_validation_summary(json.loads(completed.stdout), enforce_promotion_backtest_gates) + except (AttributeError, TypeError, ValueError, KeyError): + artifact.update(failure_stage="validation_result_invalid", numeric_execution={"status": "outcome_unknown"}) + return artifact + artifact.update(safe) + artifact.update(status="accepted", research_executed=True, numeric_execution={"status": "succeeded"}) + return artifact + + def _write(path: Path, value: Mapping[str, Any]) -> None: path.parent.mkdir(parents=True, exist_ok=True) path.write_text(json.dumps(value, sort_keys=True, separators=(",", ":")) + "\n") -def initialize_record(path: Path, context: Mapping[str, str]) -> None: +def initialize_record(path: Path, context: Mapping[str, str], *, operation: str = "soxl_learning") -> None: """Write the bound terminal placeholder before setup or remote reads begin.""" _validate_authority(context) + if operation not in {"soxl_learning", "soxl_validation"}: + raise ManualLearningError("manual_operation_invalid") _write( path, { "schema_version": ARTIFACT_SCHEMA, - "operation": "soxl_learning", + "operation": operation, "status": "parked", "failure_stage": "setup_incomplete", "authority": { @@ -759,6 +944,7 @@ def initialize_record(path: Path, context: Mapping[str, str]) -> None: def main(argv: Sequence[str] | None = None) -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--parameter-grid") + parser.add_argument("--development-summary", type=Path) parser.add_argument("--manifest-sha256") parser.add_argument("--root", type=Path) parser.add_argument("--consumer-source", type=Path) @@ -810,30 +996,42 @@ def main(argv: Sequence[str] | None = None) -> int: print(json.dumps({"status": result["status"], "operation": "soxl_watcher_learning"}, sort_keys=True)) return 0 if result["status"] == "accepted" else 2 manual_required = ( - args.parameter_grid, args.manifest_sha256, args.root, args.consumer_source, + args.manifest_sha256, args.root, args.consumer_source, args.ues_source, args.output, args.repository, args.ref, args.event_name, args.actor, args.run_id, args.run_attempt, ) if any(value is None for value in manual_required): parser.error("manual mode requires the original bounded input and authority arguments") + if args.development_summary is None and args.parameter_grid is None: + parser.error("manual learning requires a parameter grid") + if args.development_summary is not None and args.parameter_grid is not None: + parser.error("selected validation cannot accept a parameter grid") context = {key: getattr(args, key) for key in ("repository", "ref", "event_name", "actor", "run_id", "run_attempt")} + operation = "soxl_validation" if args.development_summary is not None else "soxl_learning" try: - result = run_manual_learning( - parameter_grid=args.parameter_grid, manifest_sha256=args.manifest_sha256, - root=args.root, consumer_source=args.consumer_source, ues_source=args.ues_source, - context=context, progress_writer=lambda value: _write(args.output, value), - ) + if args.development_summary is not None: + result = run_manual_validation( + development_summary=args.development_summary, manifest_sha256=args.manifest_sha256, + root=args.root, consumer_source=args.consumer_source, ues_source=args.ues_source, + context=context, progress_writer=lambda value: _write(args.output, value), + ) + else: + result = run_manual_learning( + parameter_grid=args.parameter_grid, manifest_sha256=args.manifest_sha256, + root=args.root, consumer_source=args.consumer_source, ues_source=args.ues_source, + context=context, progress_writer=lambda value: _write(args.output, value), + ) exit_code = 0 if result["status"] == "accepted" else 2 except ManualLearningError as exc: result = { - "schema_version": ARTIFACT_SCHEMA, "operation": "soxl_learning", + "schema_version": ARTIFACT_SCHEMA, "operation": operation, "status": "parked", "failure_stage": str(exc), "research_executed": False, "learning_only": True, "no_order": True, "size_zero_required": True, "promotion_eligible": False, } exit_code = 2 _write(args.output, result) - print(json.dumps({"status": result["status"], "operation": "soxl_learning"}, sort_keys=True)) + print(json.dumps({"status": result["status"], "operation": operation}, sort_keys=True)) return exit_code diff --git a/tests/test_research_input_readback_workflow.py b/tests/test_research_input_readback_workflow.py index b9c49907..05f8eabe 100644 --- a/tests/test_research_input_readback_workflow.py +++ b/tests/test_research_input_readback_workflow.py @@ -202,12 +202,12 @@ def test_workflow_keeps_readback_default_and_bounds_manual_learning() -> None: text = workflow_text() assert "github.ref == 'refs/heads/main'" in text - assert "inputs.operation != 'soxl_learning' || github.run_attempt == 1" in text + assert "inputs.operation == 'readback' || github.run_attempt == 1" in text assert "default: readback" in text assert "- soxl_learning" in text assert "repository: QuantStrategyLab/UsEquitySnapshotPipelines" in text assert "ref: ca61b82c2a508a1cc81fb5831294ba9835ac41c2" in text - assert "ref: b03ecbe4e0a7a0de22f298499f867a7039e4b60a" in text + assert "|| 'b03ecbe4e0a7a0de22f298499f867a7039e4b60a'" in text assert "ref: 7756fe32585e85cf1d09a163203a02e3eee39fe1" in text assert "uv sync --locked --no-dev --no-editable --python 3.11" in text assert text.index("name: Install the frozen validator runtime") < text.index( @@ -215,10 +215,23 @@ def test_workflow_keeps_readback_default_and_bounds_manual_learning() -> None: ) assert "trap cleanup EXIT" in text assert "if: always()" in text - assert "if: always() && inputs.operation == 'soxl_learning'" in text + assert "if: always() && inputs.operation != 'readback'" in text assert "python3 -m scripts.run_soxl_manual_learning" in text assert 'UV_PROJECT_ENVIRONMENT="$UES_ENV_ROOT" python3 -m scripts.run_soxl_manual_learning' in text assert text.index("uv run --no-sync python") < text.index( 'UV_PROJECT_ENVIRONMENT="$UES_ENV_ROOT" python3 -m scripts.run_soxl_manual_learning' ) assert "run_soxl_core_only_p3_evidence" not in text + + +def test_selected_validation_uses_original_evidence_and_an_isolated_strict_gate(): + text = workflow_text() + assert "- soxl_validation" in text + assert "QuantStrategyLab/QuantPlatformKit" in text + assert "7363011d56926d39f4fffeb036e511391114e39f" in text + assert "soxl-manual-learning-34376283866-1" in text + assert "--development-summary" in text + assert '"$VALIDATION_CONTROL_ENV/bin/python" -m scripts.run_soxl_manual_learning' in text + assert "inputs.operation == 'readback' || github.run_attempt == 1" in text + assert "actions: read" in text + assert text.index("Install the isolated strict gate") < text.index("Authenticate for this research input only") diff --git a/tests/test_run_soxl_manual_learning.py b/tests/test_run_soxl_manual_learning.py index b831b024..d108d818 100644 --- a/tests/test_run_soxl_manual_learning.py +++ b/tests/test_run_soxl_manual_learning.py @@ -627,3 +627,169 @@ def verify(active: dict[str, object]) -> dict[str, object]: other_ref = workflow_ref.replace("refs/heads/main", "refs/heads/other") with pytest.raises(PermissionError, match="workflow_ref .* not allowed"): verify(payload | {"workflow_ref": other_ref, "job_workflow_ref": other_ref, "ref": "refs/heads/other"}) + + +def promotion_validation_result() -> dict[str, object]: + from datetime import date + from quant_platform_kit.strategy_lifecycle.contracts import ( + BacktestResult, BacktestValidationIdentity, OptimizationProposal, PromotionBacktestRun, + PromotionCostModel, PurgedWalkForwardFold, + ) + from scripts import run_soxl_manual_learning as module + + profile = "soxl_soxx_three_asset_mid_weight_learning_v1" + folds = tuple(PurgedWalkForwardFold(*map(date.fromisoformat, item)) for item in ( + ("2022-12-28", "2023-06-30", "2023-07-03", "2023-12-29"), + ("2024-01-02", "2024-06-28", "2024-07-01", "2024-12-31"), + ("2025-01-02", "2025-02-28", "2025-03-03", "2025-07-31"), + )) + proposal = OptimizationProposal( + strategy_profile=profile, domain="us_equity", + current_params={"blend_gate_mid_soxl_weight": 0.65}, + proposed_params={"blend_gate_mid_soxl_weight": 0.55}, + recommendation="research_candidate", search_iterations=3, + optimization_method=f"bounded_development_tradeoff:sha256:{module.DEVELOPMENT_SUMMARY_SHA256}", + ) + + def runs(weight, role): + result = [] + for cost in (5.0, 10.0, 15.0): + def metrics(start, end, fold=None): + suffix = f"_wf{folds.index(fold)}" if fold else "_locked_oos" + result_id = f"soxl-three-asset-{module.DEVELOPMENT_SUMMARY_SHA256}-{role}-cost-{cost:g}{suffix}" + return BacktestResult( + strategy_profile=profile, domain="us_equity", + param_set_id=result_id, + params={"blend_gate_mid_soxl_weight": weight}, + cagr=weight / 2, max_drawdown=weight / 3, sharpe_ratio=1.1, + total_return=weight / 2, volatility=0.2, observation_count=100, + start_date=start, end_date=end, source_revision=UES_REVISION, + cost_model=f"all_in_per_side_{cost:g}bps", + cost_inputs={"commission_bps": 0.0, "slippage_bps": cost, "market_impact_bps": 0.0}, + validation_identity=BacktestValidationIdentity( + protocol="purged_walk_forward.v1", fold_id=result_id, + fold_role="test" if fold else "locked_oos", + train_start=fold.train_start if fold else None, + train_end=fold.train_end if fold else None, + test_start=start, test_end=end, + locked_oos_start=date(2025, 8, 4), locked_oos_end=date(2026, 8, 4), + purge_days=1, embargo_days=1, + ), + ) + result.append(PromotionBacktestRun( + strategy_profile=profile, domain="us_equity", folds=folds, + fold_results=tuple(metrics(fold.test_start, fold.test_end, fold) for fold in folds), + locked_oos_result=metrics(date(2025, 8, 4), date(2026, 8, 4)), + locked_oos_start=date(2025, 8, 4), locked_oos_end=date(2026, 8, 4), + purge_days=1, embargo_days=1, source_revision=UES_REVISION, + cost_model=PromotionCostModel(model_id=f"all_in_per_side_{cost:g}bps", commission_bps=0.0, slippage_bps=cost, market_impact_bps=0.0), + ).to_dict()) + return result + + return { + "status": "PROMOTION_BACKTEST_RUNS_BUILT", "stage": "promotion_validation", + "learning_only": True, "no_order": True, "size_zero_required": True, + "live_authority_granted": False, + "promotion_eligible": False, "proposal": proposal.to_dict(), + "baseline_promotion_runs": runs(0.65, "baseline"), + "candidate_promotion_runs": runs(0.55, "candidate"), + } + + +@pytest.mark.parametrize("path,replacement", [ + (("baseline_promotion_runs", 0, "folds", 0, "train_start"), "2022-12-29"), + (("baseline_promotion_runs", 0, "purge_days"), 2), + (("baseline_promotion_runs", 0, "fold_results", 0, "validation_identity", "train_start"), "2022-12-29"), + (("live_authority_granted",), True), + (("proposal", "walk_forward_passed"), True), + (("candidate_promotion_runs", 0, "locked_oos_result", "param_set_id"), lambda value: value + "-unbound"), + (("candidate_promotion_runs", 0, "locked_oos_result", "cost_model"), "different-cost"), + (("candidate_promotion_runs", 0, "locked_oos_result", "cost_inputs", "slippage_bps"), 999.0), +]) +def test_selected_validation_rejects_conflicting_plan_or_evidence(path, replacement): + from quant_platform_kit.strategy_lifecycle.research_promotion_cycle import enforce_promotion_backtest_gates + from scripts import run_soxl_manual_learning as module + value = promotion_validation_result() + target = value + for key in path[:-1]: + target = target[key] + target[path[-1]] = replacement(target[path[-1]]) if callable(replacement) else replacement + with pytest.raises(module.ManualLearningError): + module._strict_validation_summary(value, enforce_promotion_backtest_gates) + + +def test_selected_validation_uses_strict_gate_and_reports_tradeoff_without_ai(paths, tmp_path, monkeypatch): + from scripts import run_soxl_manual_learning as module + summary = tmp_path / "development.json" + summary.write_text('{"synthetic":true}') + monkeypatch.setattr(module, "DEVELOPMENT_SUMMARY_SHA256", module._summary_digest(json.loads(summary.read_text()))) + result = promotion_validation_result() + result["private_extra"] = "must-not-be-published" + calls = [] + progress = [] + + def run(argv): + calls.append(argv) + return SimpleNamespace(returncode=0, stdout=json.dumps(result)) + + with patch.object(module.AiGatewayClient, "execute", side_effect=AssertionError("validation must not call AI")): + output = module.run_manual_validation( + development_summary=summary, manifest_sha256="a" * 64, root=paths["root"], + consumer_source=paths["consumer"], ues_source=paths["ues"], context=context(), + command_runner=run, progress_writer=progress.append, + ) + assert len(calls) == 1 + assert "--promotion-validation-development-summary" in calls[0] + assert "--blend-gate-mid-soxl-weight" not in calls[0] + assert output["status"] == "accepted" + assert output["strict_backtest_gate"]["status"] == "passed" + assert output["promotion_eligible"] is False + assert output["human_quality_decision_required"] is True + assert output["candidate_promotion_runs"][0]["locked_oos_result"]["validation_identity"]["protocol"] == "purged_walk_forward.v1" + assert all(row["candidate_max_drawdown"] < row["baseline_max_drawdown"] for row in output["oos_comparison"]) + assert "must-not-be-published" not in json.dumps(output) + assert progress[0]["numeric_execution"]["status"] == "started" + + +def test_selected_validation_rejects_changed_source_before_execution(paths, tmp_path): + from scripts import run_soxl_manual_learning as module + summary = tmp_path / "development.json" + summary.write_text('{"changed":true}') + with pytest.raises(ManualLearningError, match="validation_development_source_invalid"): + module.run_manual_validation( + development_summary=summary, manifest_sha256="a" * 64, root=paths["root"], + consumer_source=paths["consumer"], ues_source=paths["ues"], context=context(), + command_runner=lambda _: pytest.fail("changed source must not execute"), + ) + + +@pytest.mark.parametrize("failure", ["short_oos", "changed_candidate", "nan_metrics", "timeout"]) +def test_selected_validation_parks_failures_without_retry(paths, tmp_path, monkeypatch, failure): + from scripts import run_soxl_manual_learning as module + summary = tmp_path / "development.json" + summary.write_text('{"synthetic":true}') + monkeypatch.setattr(module, "DEVELOPMENT_SUMMARY_SHA256", module._summary_digest(json.loads(summary.read_text()))) + result = promotion_validation_result() + candidate = result["candidate_promotion_runs"][0] + if failure == "short_oos": + candidate["locked_oos_end"] = "2026-07-31" + elif failure == "changed_candidate": + candidate["locked_oos_result"]["params"]["blend_gate_mid_soxl_weight"] = 0.5 + elif failure == "nan_metrics": + candidate["locked_oos_result"]["cagr"] = float("nan") + calls = [] + + def run(argv): + calls.append(argv) + if failure == "timeout": + raise subprocess.TimeoutExpired("synthetic", 1) + return SimpleNamespace(returncode=0, stdout=json.dumps(result)) + + output = module.run_manual_validation( + development_summary=summary, manifest_sha256="a" * 64, root=paths["root"], + consumer_source=paths["consumer"], ues_source=paths["ues"], context=context(), command_runner=run, + ) + assert len(calls) == 1 + assert output["status"] == "parked" + assert output["promotion_eligible"] is False + assert "oos_comparison" not in output