From ade5035d0f0cce14fef3d2fdae6fa6743ae14ac7 Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Sun, 27 Sep 2026 15:12:13 +0000 Subject: [PATCH 01/17] fix(reference): preserve mark-index stale causes for consumer recovery --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 68 ++++++++++++++++++++ qdl/reference/execution_live.py | 14 ++-- tests/test_execution_mark_index_live_view.py | 37 +++++++++++ 3 files changed, 115 insertions(+), 4 deletions(-) diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index 36e2a25b..43f6eb53 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -60124,3 +60124,71 @@ immediate reader rollback5b66c2cd retained with same KN config. Old10SQLite role stopped; state retained, not assumed replayable beyond Kafka retention. V1/VN, otherTSroles,alpha execution and order path unchanged. This final documentation receipt will follow dev->main without altering the published tag or artifact hashes. + +### Post-release TRADE Burst Recovery And Live Consumer Readiness (2026-09-27) + +Status: IN_PROGRESS; owner approved narrow TRADE recovery throughput and +MARK_INDEX rejection correction, isolated stress measurement, then live binding +readiness. Guide: Kafka-native architecture review sections 18.12-18.13 and +KN cursor v3 contracts. No new architecture or full-catalogue rerun. + +Baseline: v2.2.0 Query f7351c3b / Rust 7fe34806 / TS reader 7e7e2d02; +TS manifest10, alpha manifests14; auth environment paper. Observed 13:00-13:05UTC +TRADE replay/overflow recurrence; later TS Binance BNB/DOGE MARK_INDEX +SOURCE_UNAVAILABLE. Earlier bounded acceptance is historical, not burst proof. +Owner plan edits elsewhere must be preserved. + +Scope: exact rejected-view diagnosis, bounded consumption/replay correctness, +shared Binance/OKX five-symbol regression, isolated real-capture stress with +throughput/queue/recovery/latency/resource evidence. Do not increase buffers, +relax freshness, drop durable trades, enable orders, reset state or broad-prune. +Record exact role/image rollback packet before runtime replacement. Live +identities stay inactive until affected acceptance passes; environment/cursor +fencing cannot be bypassed by renaming paper. + +Exit: regress apply/ACK failure, cancellation, reconnect/replay and MARK_INDEX +stale/disconnect/generation; isolated stress preserves identity/order and drains +bounded queues after burst; measure actual consumer readiness and request latency +separately from event-to-cache age. In-scope failures remain open. Cleanup only +new disposable resources; preserve active and named rollback artifacts. + + +Source correction checkpoint: MARK/INDEX private reader collapsed COMPONENT_STALE +and SESSION_LIVENESS into SOURCE_UNAVAILABLE. Preserve bounded cause, classify +these as DATA_STALE through existing query mapping; actual disconnect/fence/gap +remain fail-closed.47 targeted live-view/latency/lineage tests PASS(network none). +Read-only actual-cache diagnostic:12 reads each of10 MARK/INDEX products,0rejects +in this short window; this does NOT prove the earlier transient is eliminated. +The first diagnostic used InstrumentIdentity.native_symbol and stopped; corrected +to InstrumentRecord.native_symbol. Initial runtime image had no pytest; tests used +the existing TS test-capable image, no packages installed into runtime. + +TS source removes serialization/revalidation round-trip for its internal dual +market/execution projection, preserving external raw validation. Polling MARK +keeps failure health blocked but retries transient typed failures on its existing +cadence rather than escalating stream backoff; no stale cache rewrite.252 TS +consumer/bridge/projector/health regressions PASS including five symbols/two venues, +ACK failure and paired projection parity. Source-only; no runtime promotion yet. + +Isolated capture: md.canonical.v2 read_committed manual assign, auto commit/store +both false, no consumer-group join or offset mutation. Capture5543real TRADE rows +covers10products; SHA078fa7602f5d2b5077e9401ef3811ddda602adc6eda57958ebe36e12ce107599. +Typed templates fetched with existing TS identity,10products. Files private outside +Git at ~/.local/state/qdl-v2/burst-recovery-20260927. Test-clock replay preserves +provider bytes/timestamps and is not a live freshness certificate. + +Same 1CPU client/0.5CPU isolated Redis: legacy2668.9event/s vs paired3067.8event/s, +ordered execution payload digest identical e9c3a62e3a69aba1cf0232438c3294fceaf60f042b9fb0a947d94073e049c244. +Open-loop paired2000event/s,20test-only capture repetitions:110860offered/received, +0overflow,queue peak1batch(of16),55.431s,CPU38.845s,producer max lateness19.28ms. +This isolates projection+Redis pubsub, NOT gRPC/replay/ACK end-to-end. Do not infer +production burst closure or live identity activation from this component test. +Runtime stays v2.2.0; live mapping and full stream recovery stress remain open. + +Cleanup checkpoint: isolated ts-burst-redis stopped/removed; disposable clients +used --rm. Docker inventory confirms no burst test containers remain. No images +built, no cache/image prune and no shared volume cleanup. Post-check disk free +112754847744 bytes; no pre-clean measurement, so no reclaimed-byte claim. +Query1/2, Stream1/2 and market_data_service retain original image/start times, +restart0/OOMfalse. Live mapping inactive. Full gRPC/replay/ACK recovery stress +and affected runtime acceptance remain implementation work, not certified debt. diff --git a/qdl/reference/execution_live.py b/qdl/reference/execution_live.py index 2525e47c..c2ac59a7 100644 --- a/qdl/reference/execution_live.py +++ b/qdl/reference/execution_live.py @@ -208,7 +208,8 @@ async def fetch( request, capability, code, - "execution MARK/INDEX live view did not return a current active record", + "execution MARK/INDEX live view did not return a current active record: " + + ",".join(dict.fromkeys(errors)), ) def stats(self) -> dict[str, int]: @@ -236,8 +237,13 @@ def _bounded_reason(response: httpx.Response) -> str: except (ValueError, AttributeError): return "UNAVAILABLE" normalized = str(detail).upper() - if normalized.endswith(":STALE"): - return "STALE" + # Component cadence and session age are freshness failures, not an + # absent provider. Preserve the bounded reason for consumer diagnosis. + reason = normalized.rsplit(":", 1)[-1] + if reason in {"STALE", "COMPONENT_STALE", "SESSION_LIVENESS"}: + return reason + if reason in {"SESSION_STATE", "QUIET_POLICY_UNAVAILABLE", "LINEAGE_INVALID"}: + return reason if normalized.endswith(":GAP_OR_RESYNC"): return "GAPPED" if normalized.endswith(":IDENTITY_MISMATCH"): @@ -250,7 +256,7 @@ def _bounded_reason(response: httpx.Response) -> str: @staticmethod def _failure_code(errors: list[str]) -> str: - if "STALE" in errors: + if any(reason in {"STALE", "COMPONENT_STALE", "SESSION_LIVENESS"} for reason in errors): return "LIVE_VIEW_STALE" if "GAPPED" in errors: return "LIVE_VIEW_GAPPED" diff --git a/tests/test_execution_mark_index_live_view.py b/tests/test_execution_mark_index_live_view.py index 30328ef9..b74227c8 100644 --- a/tests/test_execution_mark_index_live_view.py +++ b/tests/test_execution_mark_index_live_view.py @@ -1239,6 +1239,43 @@ async def handler(request: httpx.Request) -> httpx.Response: self.assertEqual(result.status, ReferenceStatus.ERROR) self.assertEqual(result.error_code, "LIVE_VIEW_PROTOCOL") + async def test_reader_preserves_typed_quiet_rejection_without_retry_or_rest(self): + cases = { + "STALE": "LIVE_VIEW_STALE", + "COMPONENT_STALE": "LIVE_VIEW_STALE", + "SESSION_LIVENESS": "LIVE_VIEW_STALE", + "SESSION_STATE": "LIVE_VIEW_UNAVAILABLE", + "QUIET_POLICY_UNAVAILABLE": "LIVE_VIEW_UNAVAILABLE", + "LINEAGE_INVALID": "LIVE_VIEW_UNAVAILABLE", + "GAP_OR_RESYNC": "LIVE_VIEW_GAPPED", + "IDENTITY_MISMATCH": "LIVE_VIEW_IDENTITY", + "FENCED": "LIVE_VIEW_UNAVAILABLE", + } + for reason, expected in cases.items(): + with self.subTest(reason=reason): + calls = [] + async def handler(request): + calls.append(request.url.host) + return httpx.Response(409, json={ + "detail": "execution MARK/INDEX live view unavailable:" + reason, + }, request=request) + async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as client: + reader = HttpExecutionMarkIndexReader( + ("https://stream_v2_active:8200",), SECRET, client=client, + ) + result = await reader.fetch( + ReferenceRequest(self.record, ReferenceProduct.MARK_INDEX_PRICE), + max_freshness_ms=2000, source_policy_id="crypto_liquid_v2", + event_recency_policy=StalePolicy.OBSERVE, + max_session_liveness_ms=45000, + ) + self.assertEqual(result.status, ReferenceStatus.ERROR) + self.assertEqual(result.error_code, expected) + self.assertEqual(calls, ["stream_v2_active"]) + self.assertTrue(result.error_detail.endswith(reader._bounded_reason( + httpx.Response(409, json={"detail": "unavailable:" + reason}) + ))) + async def test_reader_uses_one_deadline_across_active_passive_urls(self): envelope = _envelope(self.binding, sequence=50) payload = { From 3436b8e42a2b9b1db66f7f538702a18635a33560 Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Sun, 27 Sep 2026 15:55:43 +0000 Subject: [PATCH 02/17] docs: record reader recovery rollout and thirty-minute observation --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 104 ++++++++++++++++++++++ 1 file changed, 104 insertions(+) diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index 43f6eb53..52d35b33 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -60192,3 +60192,107 @@ built, no cache/image prune and no shared volume cleanup. Post-check disk free Query1/2, Stream1/2 and market_data_service retain original image/start times, restart0/OOMfalse. Live mapping inactive. Full gRPC/replay/ACK recovery stress and affected runtime acceptance remain implementation work, not certified debt. + +Continuation: owner approved end-to-end recovery and runtime acceptance plus a +30-minute read-only observer subagent. Narrow packet: Query KN1/KN2 overlay +ade5035 on f7351c3b, TS market_data overlay75df1c4 on7e7e2d02; build/tests first, +record full resulting digests before rolling. Preserve configs/mounts/TLS, +Rust/Kafka/cache/V1/alpha/order path. Rollback exactly the replaced role images. +No live mapping until recovery and actual reader acceptance pass. Client fault +tests own isolated Redis/cursors; never pause shared production consumers. + +Runtime packet frozen before action: private burst-recovery-20260927/packet +query-candidate/rollback.compose.json and ts-candidate/rollback.compose.json. +Only image fields change: query_kn_1/query_kn_2 +sha256:f7351c3bda080a6d1266d49480dc65d7e590375f178252f858f87a04b1a4d685 -> +sha256:3dd54bfd8b9de0b143d317798ab9c63cebc502d01fe3cf6f513032d6484b5b11; +market_data (container market_data_service) +sha256:7e7e2d026a9f301db0c32237063b4f518a6503e986e6206ad9a7e1a79b01ceea -> +sha256:1d86d3e820795a50b2539b09a6d35b102efd461e7bd7dec1801730a5daffdc46. +Exact patched-file parents equal deployed files byte-for-byte. Packaged TS +252 tests PASS; first run had5 fixture mount path failures, corrected mounts. +Query normal import path passes and patched module SHA matches source. Direct +execution_live import exposes existing package cycle; normal query entry imports +query first. Initial FROM sha256 syntax could not resolve in BuildKit; corrected +to existing local tag and verified resolved digest. No dependency/network changes. +Roll Query1 then healthy, Query2 then healthy; then TS market_data only. Stop and +rollback affected role on startup error or persistent new unhealthy regression. +Retain same revision10/SDK2.0.5/config/network/mounts; no live activation yet. + +Runtime continuation evidence (2026-09-27): Query KN1/KN2 rolled sequentially +15:23UTC, TS market_data15:24UTC, same mounts/config/revision10/SDK2.0.5. No +other role recreated. Candidate images3dd54bfd/1d86d3e8 remain active; rollback +f7351c3b/7e7e2d02 retained. New source scripts accept_market_trade_recovery.py +and measure_v2_binding_reads.py in TS provide repeatable no-order measurements. + +Fullchain client uses actual Rust gRPC, TS facade/projection, isolated Redis and +SDK fsynced ACK. An independent stream starting at the same signed cursor is +compared through the final durable ACK; matching physical offset gaps are legal. +Initial20additionalstreams exceeded TS identity50stream quota (production40), +so OKX was refused; all5Binance prefixes matched. Fixed harness to one venue, +10additionalstreams, not quota relaxation. First OKX run was invalidated by my +premature stop of test-only Redis; errors kept in recovery-okx-interrupted.json. +Rerun OKX PASS5/5,8s inducedclient loss, recovered8.126-10.893s including pause; +Binance harder30s pause PASS5/5, recovered30.362-30.778s, exact12336records through +ACK,0cursorerrors. At-least-once reconnect may replay unapplied/unacknowledged +prefix; not an exactly-once broker-event publication certificate. Actual market +rate during these runs is not an accelerated2000/s end-to-end capacity proof. +Seven harness regressions PASS missing/changed/extra/empty-prefix and percentile +labelling. No production Kafka/Redis/cursor writes from fault clients. + +Separate external read-only TS Redis observer300.027s:160169notifications, +60feed/product groups,0cache missing/identity/watermark/future-time errors. +Request->typed usable SDK matrix120reads across2Queryreplicas:116usable, +4TRADEexecutioneligibility refusals retained, no MARK/QUOTE/L2/BAR refusals. +Per-feed n20(orTRADE16) is insufficient forp99. TargetedTRADE follow-up20reads, +18usable/2refused: exact rejected quality LIVE/sessionLIVE/gapfalse/complete, +eventSTALE at3126/3400ms vs3000ms. Do not call quiet last-trade execution usable. +Evidence at burst-recovery-20260927/{fullchain,cache-observer.json,observer-30m}. + +Live activation boundary checked in code, not assumed from identity names: +current TS manifest execution_dependency=PAPER_ONLY, environment=paper; +Query stable.py:529 and Rust auth.rs:302 enforce one configured auth environment. +KN cache LPK/cursor also includes environment. Live credentials cannot simply +be renamed/reused or accepted by relaxing environment verification. Existing +mainnet market bytes and tested TS60 do not certify a new live auth realm. +Live mapping remains inactive pending coherent identity/realm/cursor binding; +no widened authorization or automatic order enablement was performed. + +Closure preparation:259 focused TS tests PASS, Ruff changed scripts/tests PASS. +Disposed only ts-burst-cache, ts_burst_recovery_test network and --rm clients; +removed temporary client credential envs, test-only cursor directories and two +build contexts after hashing. Retained all market capture/receipt evidence. +Pruned exactly4 unshared source.local BuildKit records from the two15:20builds: +exxu40ud2r41aynmldcnw69lg,s1w22i9qi9uqketb116vdgvf6, +7xpedbd4elu2wq3lcernqwfm7,zfpet5gsafuvnqid1udyldlvm(~180kB reported). +Filesystem available112067633152->112331137024bytes; other host writes/retention +continue, so this delta is not attributed solely to cleanup. Both new images +are active, original two are exact rollbacks; no test-only image left from this +slice, no broad prune or old production-state removal. DataLayer has one checkout; +TS existing Portal/upgrade worktrees preserved, none newly created. + +Final bounded observation:1800s,2026-09-27T15:20:28Z..15:50:28Z,61samples; +after TS rollout52samples,51READY including startup,48/48post-startupREADY60/60. +One startup sample not-ready,0fallback/0v2errors after startup. Stream717401live +and4675replayed events (includes isolatedclients),0newoverflow/reader/decode/ +duplicate errors,queueend0.35quota refusals belonged to first oversized test; +no production TS session degradation there. No unplanned restart/OOM. +MARK reference refresh rejection counter0 on new TSreader. Execution readiness +is separate: final telemetry50/60execution-ready vs60/60session-ready, not a +claim every TRADE last price is usable. Known fiveBinance3d history source +refusals remain outside this correction; no fabricated bars or broadened cert. + +Resource mean DataLayer4.47665vCPU excluding TS/V1,TS0.72865vCPU; observed DL +mean within5core but not a new maximum-capacity certificate. TS cgroup peak186MiB; +no cap changes. Ingestor throttled-period fraction6.82%/8.16%, not percent CPU lost. +StableRedis dockerlogs timed out; bounded direct read of configured json-file +log confirms file0bytes. Health/cgroup accessible; do not claim unseen log proof. +Acceptance-index SHA398fb34a889f10b0e8710fbc51cbf2e119e79039393558d9fb4aabcc96be43f3. + +Recorded status: BOUNDED_READER_RECOVERY_RUNTIME_ACCEPTED / LIVE_MAPPING_INACTIVE. +No full accelerated gRPC burst capacity claim;2000/s was projection+Redis only. +Overall live-consumer task remains IN_PROGRESS due unimplemented coherent live +identity/realm mapping, not a request to loosen gates or repeat full C2. Current +TS paper/no-order reader can use the accepted correction. Published release +remainsv2.2.0; these immutable overlays are not a newly published stable tag. +No push/merge, no alpha/order activation. Main plans preserve owner edits. From 609bb3225619c348cb1c07a7e1e72f10006817b3 Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Sun, 27 Sep 2026 16:49:35 +0000 Subject: [PATCH 03/17] Bind public-market workload keys to paper sandbox and live realms --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 53 ++++++++ qdl/consumer/manifest.py | 2 + qdl/runtime/stable.py | 7 +- qdl/security/data_plane.py | 18 +++ qdl/security/policy.py | 13 +- rust/qdl-stream-gateway/src/auth.rs | 57 ++++++++- rust/qdl-stream-gateway/src/authority.rs | 3 +- rust/qdl-stream-gateway/src/main.rs | 6 +- .../qdl-stream-gateway/tests/native_stream.rs | 1 + scripts/compile_consumer_realms.py | 83 +++++++++++++ tests/test_consumer_key_realms.py | 115 ++++++++++++++++++ ...YER_V2_KAFKA_NATIVE_ARCHITECTURE_REVIEW.md | 22 ++++ 12 files changed, 374 insertions(+), 6 deletions(-) create mode 100644 scripts/compile_consumer_realms.py create mode 100644 tests/test_consumer_key_realms.py diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index 52d35b33..13f6d8f1 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -60296,3 +60296,56 @@ identity/realm mapping, not a request to loosen gates or repeat full C2. Current TS paper/no-order reader can use the accepted correction. Published release remainsv2.2.0; these immutable overlays are not a newly published stable tag. No push/merge, no alpha/order activation. Main plans preserve owner edits. + +#### Approved continuation: native burst and live read identities + +Status: IN_PROGRESS. Owner requests full gRPC burst/recovery through 4000 events/s +and actual live mapping, not component-only throughput or renamed paper keys. +Continue under KN guide 18.12-18.13; preserve provider bytes, freshness, ACK-after- +apply, realm/cursor fences, existing paper access and all order services. +Tests: isolated captured-provider native gRPC -> SDK -> projection -> Redis -> +durable ACK, offered/completed accounting, bounded queue/RAM, disconnect/replay, +slow consumer isolation and drain. Report capacity only at measured rates. +Live credentials require explicit consumer entitlement, independent identity, +realm-bound auth/cursors and no-order reads. Journal exact runtime image/config +and per-role rollback before activation; do not reset shared market history. +Inspect known Binance 3d native historical window rejection without fabricated +repair. Source fixes require targeted regressions; no unrelated catalogue rerun. +Cleanup only this continuation's isolated resources; retain active/rollback images. + +Live-realm source edit was rejected by the automatic approval reviewer before +execution: widening authentication needs explicit key/consumer/realm proof. +No authentication file or runtime was changed. A narrow clarification was sent +for per-key paper/live realm binding, unchanged subject/revision/consumer checks, +shared public-market storage realm and consumer-bound cursor. Do not bypass this +rejection. Continue unaffected isolated burst and native history investigation. + +Owner explicitly approved narrow key-to-realm authentication packet and added +sandbox mode. Implement exact pinned key realms paper/sandbox/live, with existing +subject/consumer/revision and mTLS checks; common public-market storage namespace +and consumer-bound cursor remain unchanged. No per-request network auth lookup, +new service, order rights or silent credential reuse. Regress wrong realm/key/ +consumer/revision, old paper compatibility and cross-consumer cursor refusal. + +Source checkpoint: explicit JWT key realms implemented for paper/sandbox/live in +Python Query and Rust Stream; legacy single-realm config remains strict. Realm +rotation participates in Stream reauthorization. Public data LPK and signed +cursor storage realm remain unchanged; token subject/consumer/revision and cursor +consumer checks remain required. No new service or per-request network lookup. +Focused Python auth/compiler suite23PASS. Rust worker reports30unit+33native RPC +PASS (artifact receipt pending integration). Compiler scope restricted to exactly +TS/Binance-alpha/OKX-alpha approved IDs; FORBIDDEN alpha direct execution dependency +is preserved, not promoted. Initial compiler rejected FORBIDDEN alpha; fixed with +explicit approved-ID/policy map rather than a permissive generic promotion. +A Python3.10 invocation of KN backend tests failed import StrEnum; runtime is +Python3.12. Query integration tests require disposable Redis, skip without it; +skips are not acceptance. No live identity/config activated at this checkpoint. + +Auth slice verification: Python3.12 runtime image unittest auth/contract/async +cursor suite45PASS. Compiler now excludes deferred VN policy explicitly; TS +sandbox/live each60crypto requirements, alpha Binance/OKX each375. Six draft +manifests compiled outside Git, not active. Unit matrix covers paper/sandbox/live, +wrong key/realm/consumer/revision, unknown mappings, preserved quotas/purposes, +no implicit other-consumer promotion, and FORBIDDEN alpha authority preservation. +Ruff targeted security/compiler/tests PASS with --isolated (repository Poetry +requires-python caret is rejected by installed Ruff TOML parser otherwise). diff --git a/qdl/consumer/manifest.py b/qdl/consumer/manifest.py index 8b14decd..33d4c5fa 100644 --- a/qdl/consumer/manifest.py +++ b/qdl/consumer/manifest.py @@ -102,6 +102,8 @@ def __post_init__(self) -> None: raise ValueError("consumer manifest contains an unknown data-plane permission") if self.execution_dependency not in {"FORBIDDEN", "PAPER_ONLY", "ALLOWED"}: raise ValueError("consumer execution dependency policy is invalid") + if self.environment in {"sandbox", "live"} and self.execution_dependency == "PAPER_ONLY": + raise ValueError("sandbox/live consumer cannot declare PAPER_ONLY execution dependency") if not self.requirements: raise ValueError("consumer manifest requires at least one data requirement") if self.rollback_contract not in {"V1", "V2"}: diff --git a/qdl/runtime/stable.py b/qdl/runtime/stable.py index 33e9e85a..6c66a69e 100644 --- a/qdl/runtime/stable.py +++ b/qdl/runtime/stable.py @@ -526,7 +526,12 @@ def stable_grpc_server_credentials( def load_stable_manifests(config: StableRuntimeConfig) -> ConsumerManifestRegistry: manifests = tuple(ConsumerManifestLoader.load(path) for path in config.manifest_paths) - if any(item.environment != config.environment for item in manifests): + # Public market storage/cursors keep their realm. Consumer realms require + # explicit, separately pinned JWT key/subject bindings. + raw_realms = os.environ.get("QDL_DATA_JWT_KEY_ENVIRONMENTS_JSON") + realms = (DataPlaneSecurityConfig.from_environment().consumer_environments + if raw_realms else frozenset({config.environment})) + if any(item.environment not in realms for item in manifests): raise ValueError("stable consumer manifest environment mismatch") return ConsumerManifestRegistry(manifests) diff --git a/qdl/security/data_plane.py b/qdl/security/data_plane.py index 69440117..b8636591 100644 --- a/qdl/security/data_plane.py +++ b/qdl/security/data_plane.py @@ -43,6 +43,7 @@ class DataPlaneSecurityConfig: algorithms: tuple[str, ...] max_token_lifetime_seconds: int = 900 subjects_by_key_id: Mapping[str, str] | None = None + environments_by_key_id: Mapping[str, str] | None = None def __post_init__(self) -> None: if not all((self.environment.strip(), self.issuer.strip(), self.audience.strip())): @@ -60,11 +61,26 @@ def __post_init__(self) -> None: ): raise ValueError("data-plane JWT key-subject bindings are invalid") + if self.environments_by_key_id is not None: + if (not isinstance(self.environments_by_key_id, Mapping) + or self.subjects_by_key_id is None + or set(self.environments_by_key_id) != set(self.keys_by_id)): + raise ValueError("JWT key-realm bindings require exact key-subject coverage") + if any(value not in {"paper", "sandbox", "live"} for value in self.environments_by_key_id.values()): + raise ValueError("JWT consumer realm must be paper, sandbox or live") + + @property + def consumer_environments(self) -> frozenset[str]: + if self.environments_by_key_id is None: + return frozenset({self.environment}) + return frozenset(self.environments_by_key_id.values()) + @classmethod def from_environment(cls) -> "DataPlaneSecurityConfig": try: keys = json.loads(os.environ["QDL_DATA_JWT_KEYS_JSON"]) key_subjects = json.loads(os.environ["QDL_DATA_JWT_KEY_SUBJECTS_JSON"]) + realms = json.loads(os.environ.get("QDL_DATA_JWT_KEY_ENVIRONMENTS_JSON", "null")) issuer = os.environ["QDL_DATA_JWT_ISSUER"] audience = os.environ["QDL_DATA_JWT_AUDIENCE"] except (KeyError, json.JSONDecodeError) as error: @@ -82,6 +98,7 @@ def from_environment(cls) -> "DataPlaneSecurityConfig": ) return cls( environment=os.environ.get("QDL_ENVIRONMENT", "paper").lower(), + environments_by_key_id=realms, issuer=issuer, audience=audience, keys_by_id={str(key): str(value) for key, value in keys.items()}, @@ -323,6 +340,7 @@ def __init__( keys_by_id=config.keys_by_id, algorithms=config.algorithms, max_lifetime_seconds=config.max_token_lifetime_seconds, + environments_by_key_id=config.environments_by_key_id, ) def authenticate(self, bearer_token: str, *, consumer_id: str) -> DataPlaneAccess: diff --git a/qdl/security/policy.py b/qdl/security/policy.py index 470e6224..54bfb3f4 100644 --- a/qdl/security/policy.py +++ b/qdl/security/policy.py @@ -60,6 +60,7 @@ def __init__( keys_by_id: Mapping[str, str | bytes], algorithms: Sequence[str] = ("RS256", "ES256"), max_lifetime_seconds: int = 900, + environments_by_key_id: Mapping[str, str] | None = None, ): if not issuer or not audience or not keys_by_id: raise ValueError("issuer, audience and at least one verification key are required") @@ -70,6 +71,12 @@ def __init__( self._keys = dict(keys_by_id) self._algorithms = tuple(algorithms) self._max_lifetime = max_lifetime_seconds + if environments_by_key_id is not None: + if set(environments_by_key_id) != set(keys_by_id) or any( + value not in {"paper", "sandbox", "live"} for value in environments_by_key_id.values() + ): + raise ValueError("JWT key-realm bindings must cover exactly the keyring") + self._environments_by_key_id = environments_by_key_id def verify(self, token: str, *, expected_environment: str) -> Principal: try: @@ -97,7 +104,11 @@ def verify(self, token: str, *, expected_environment: str) -> Principal: if expires_at <= issued_at or expires_at - issued_at > self._max_lifetime: raise PermissionError("workload token lifetime exceeds policy") environment = str(claims["environment"]) - if environment != expected_environment: + trusted_environment = ( + self._environments_by_key_id[key_id] + if self._environments_by_key_id is not None else expected_environment + ) + if environment != trusted_environment: raise PermissionError("workload token environment mismatch") roles = frozenset(str(role) for role in claims.get("roles", [])) unknown = roles - _ROLE_PERMISSIONS.keys() diff --git a/rust/qdl-stream-gateway/src/auth.rs b/rust/qdl-stream-gateway/src/auth.rs index bd684a7c..425f7a57 100644 --- a/rust/qdl-stream-gateway/src/auth.rs +++ b/rust/qdl-stream-gateway/src/auth.rs @@ -61,6 +61,7 @@ pub struct JwtConfig { pub algorithms: Vec, pub max_lifetime_seconds: i64, pub subjects_by_key_id: BTreeMap, + pub environments_by_key_id: BTreeMap, } const OID_RSA_ENCRYPTION: &[u8] = &[0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, 0x01, 0x01, 0x01]; @@ -131,6 +132,31 @@ pub fn public_key_from_pem(pem: &str) -> Result<(Algorithm, DecodingKey), String } impl JwtConfig { + /// Explicit key realms share public market storage, never consumer identity. + /// Missing mapping retains the single-realm compatibility behavior. + pub fn with_key_environments_json(mut self, raw: &str) -> Result { + let mapping: Option> = + serde_json::from_str(raw).map_err(|error| error.to_string())?; + if let Some(mapping) = mapping { + if mapping.keys().ne(self.keys.keys()) + || mapping + .values() + .any(|value| !matches!(value.as_str(), "paper" | "sandbox" | "live")) + { + return Err("JWT key-realm bindings must cover exactly the keyring".into()); + } + self.environments_by_key_id = mapping; + } + Ok(self) + } + + pub fn key_environment(&self, key_id: &str) -> &str { + self.environments_by_key_id + .get(key_id) + .map(String::as_str) + .unwrap_or(&self.environment) + } + /// Same inputs as `DataPlaneSecurityConfig.from_environment`: a kid -> /// PEM public key map and a kid -> subject map that must cover it. pub fn from_json( @@ -182,6 +208,7 @@ impl JwtConfig { algorithms, max_lifetime_seconds, subjects_by_key_id: subjects, + environments_by_key_id: BTreeMap::new(), }) } } @@ -299,7 +326,7 @@ pub fn verify_token(config: &JwtConfig, token: &str, now: f64) -> Result = claims["roles"] @@ -565,6 +592,7 @@ mod tests { fn config(decoding: DecodingKey) -> JwtConfig { JwtConfig { + environments_by_key_id: BTreeMap::new(), environment: "paper".into(), issuer: "https://identity.test".into(), audience: "qdl-v2-stable".into(), @@ -589,6 +617,33 @@ mod tests { }) } + #[test] + fn pinned_key_realms_do_not_cross_paper_sandbox_live() { + for realm in ["paper", "sandbox", "live"] { + let (encoding, decoding) = keypair(); + let config = config(decoding) + .with_key_environments_json(&serde_json::json!({"k1": realm}).to_string()) + .unwrap(); + let now = now_seconds().floor() as i64; + for claimed in ["paper", "sandbox", "live"] { + let mut value = claims(now); + value["environment"] = claimed.into(); + let result = verify_token(&config, &token(&encoding, "k1", value), now as f64); + assert_eq!(result.is_ok(), realm == claimed); + } + } + } + + #[test] + fn key_realm_mapping_requires_exact_keys_and_known_realms() { + for raw in ["{}", r#"{"other":"live"}"#, r#"{"k1":"*"}"#, "[]"] { + let (_, decoding) = keypair(); + assert!(config(decoding).with_key_environments_json(raw).is_err()); + } + let (_, decoding) = keypair(); + assert!(config(decoding).with_key_environments_json("null").is_ok()); + } + #[test] fn a_valid_token_verifies_and_every_claim_rule_refuses() { let (encoding, decoding) = keypair(); diff --git a/rust/qdl-stream-gateway/src/authority.rs b/rust/qdl-stream-gateway/src/authority.rs index 1bfccb29..129520a2 100644 --- a/rust/qdl-stream-gateway/src/authority.rs +++ b/rust/qdl-stream-gateway/src/authority.rs @@ -101,7 +101,8 @@ pub fn reauthorize( purpose: &str, entitlement: &Entitlement, ) -> Result<(), AccessError> { - if !authority.jwt.keys.contains_key(&principal.key_id) + if authority.jwt.key_environment(&principal.key_id) != principal.environment + || !authority.jwt.keys.contains_key(&principal.key_id) || authority.jwt.subjects_by_key_id.get(&principal.key_id) != Some(&principal.subject) { return Err(AccessError::Unauthenticated( diff --git a/rust/qdl-stream-gateway/src/main.rs b/rust/qdl-stream-gateway/src/main.rs index afd2ac5e..d82476f3 100644 --- a/rust/qdl-stream-gateway/src/main.rs +++ b/rust/qdl-stream-gateway/src/main.rs @@ -134,7 +134,8 @@ fn jwt_config(environment: &str) -> Result { value["max_lifetime_seconds"] .as_i64() .ok_or("jwt config max_lifetime_seconds must be an integer")?, - ); + )? + .with_key_environments_json(&value["environments"].to_string()); } JwtConfig::from_json( environment, @@ -144,7 +145,8 @@ fn jwt_config(environment: &str) -> Result { &env("QDL_DATA_JWT_KEY_SUBJECTS_JSON")?, &env_or("QDL_DATA_JWT_ALGORITHMS", "RS256,ES256"), parsed("QDL_DATA_JWT_MAX_LIFETIME_SECONDS", "900")?, - ) + )? + .with_key_environments_json(&env_or("QDL_DATA_JWT_KEY_ENVIRONMENTS_JSON", "null")) } fn load_authority() -> Result { diff --git a/rust/qdl-stream-gateway/tests/native_stream.rs b/rust/qdl-stream-gateway/tests/native_stream.rs index 6db82b2f..63f78a79 100644 --- a/rust/qdl-stream-gateway/tests/native_stream.rs +++ b/rust/qdl-stream-gateway/tests/native_stream.rs @@ -453,6 +453,7 @@ fn keypair() -> Keys { fn jwt_config(keys: &[(&str, &Keys)]) -> JwtConfig { JwtConfig { + environments_by_key_id: BTreeMap::new(), environment: "paper".into(), issuer: "https://identity.test".into(), audience: "qdl-v2-stable".into(), diff --git a/scripts/compile_consumer_realms.py b/scripts/compile_consumer_realms.py new file mode 100644 index 00000000..a4b4fcde --- /dev/null +++ b/scripts/compile_consumer_realms.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 +"""Compile explicit sandbox/live read manifests from approved stable scopes. + +No credentials, activation or runtime mutation. The source paper manifests stay +unchanged; output belongs in a versioned runtime packet, not the source tree. +""" +from __future__ import annotations + +import argparse +import copy +import json +from pathlib import Path + +import yaml + +from qdl.consumer.manifest import ConsumerManifestLoader + + +def compile_realm(payload: dict, realm: str) -> dict: + source = ConsumerManifestLoader.from_mapping(payload) + if source.environment != "paper" or realm not in {"sandbox", "live"}: + raise ValueError("explicit paper -> sandbox/live public-market mapping required") + approved = { + "trading-system.paper.stable": "PAPER_ONLY", + "alpha.binance.paper.stable": "FORBIDDEN", + "alpha.okx.paper.stable": "FORBIDDEN", + } + if approved.get(source.consumer_id) != source.execution_dependency: + raise ValueError("only the three owner-approved stable consumer scopes may be mapped") + if source.consumer_id.count(".paper.") != 1 or not source.subject.startswith("spiffe://qdl/paper/"): + raise ValueError("stable consumer identity convention mismatch") + result = copy.deepcopy(payload) + metadata = result["metadata"] + metadata.update(id=source.consumer_id.replace(".paper.", "." + realm + "."), + subject=source.subject.replace("spiffe://qdl/paper/", "spiffe://qdl/" + realm + "/", 1), + environment=realm, revision=1) + # Alpha research permission is NOT promoted into direct Risk authority. + result["spec"]["execution_dependency"] = ( + "ALLOWED" if source.execution_dependency == "PAPER_ONLY" else "FORBIDDEN" + ) + # VN stays on its separately deferred V1 path. This packet authorizes only + # the existing crypto policies; future policies require explicit inclusion. + crypto_policies = {"crypto_primary_v2", "crypto_liquid_v2"} + result["spec"]["requirements"] = [ + row for row in result["spec"]["requirements"] + if row["source_policy_id"] in crypto_policies + ] + target = ConsumerManifestLoader.from_mapping(result) + expected = tuple(row for row in source.requirements if row.source_policy_id in crypto_policies) + if target.requirements != expected or target.quotas != source.quotas: + raise AssertionError("realm mapping must preserve approved crypto scope and quotas") + return result + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--source", type=Path, nargs="+", required=True) + parser.add_argument("--output-dir", type=Path, required=True) + args = parser.parse_args() + if args.output_dir.exists(): + raise ValueError("new output directory required; no overwrite of sealed manifests") + outputs = [] + for path in args.source: + payload = yaml.safe_load(path.read_text()) + for realm in ("sandbox", "live"): + mapped = compile_realm(payload, realm) + parsed = ConsumerManifestLoader.from_mapping(mapped) + outputs.append((parsed.consumer_id + ".yaml", mapped, parsed)) + if len({name for name, _, _ in outputs}) != len(outputs): + raise ValueError("duplicate target consumer") + args.output_dir.mkdir(parents=True, mode=0o750) + receipt = [] + for name, mapped, parsed in outputs: + (args.output_dir / name).write_text(yaml.safe_dump(mapped, sort_keys=False)) + receipt.append(dict(consumer_id=parsed.consumer_id, environment=parsed.environment, + subject=parsed.subject, manifest_revision=parsed.manifest_revision, + manifest_sha256=parsed.manifest_sha256, requirements=len(parsed.requirements))) + print(json.dumps({"status": "COMPILED_NOT_ACTIVATED", "market_storage_realm": "paper", + "order_permissions": False, "manifests": receipt}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/tests/test_consumer_key_realms.py b/tests/test_consumer_key_realms.py new file mode 100644 index 00000000..3cc247ea --- /dev/null +++ b/tests/test_consumer_key_realms.py @@ -0,0 +1,115 @@ +"""Public market storage is shared; workload keys/subjects/realms are not.""" +import unittest +from dataclasses import replace + +from qdl.consumer.manifest import ConsumerManifestLoader, ConsumerManifestRegistry +from qdl.security.data_plane import DataPlaneAccessError, DataPlaneIdentityService, DataPlaneSecurityConfig +from tests.phase7_support import TEST_AUDIENCE, TEST_ISSUER, make_token, manifest_mapping + + +class ConsumerKeyRealmTests(unittest.TestCase): + def setUp(self): + self.realms = ("paper", "sandbox", "live") + self.manifests = {} + for realm in self.realms: + mapping = manifest_mapping(consumer_id="ts." + realm, + subject="spiffe://qdl/" + realm + "/ts", instrument_uid="instrument-test") + mapping["metadata"]["environment"] = realm + mapping["spec"]["execution_dependency"] = "ALLOWED" + self.manifests[realm] = ConsumerManifestLoader.from_mapping(mapping) + self.config = DataPlaneSecurityConfig(environment="paper", issuer=TEST_ISSUER, + audience=TEST_AUDIENCE, algorithms=("HS256",), + keys_by_id={r: ("test-only-" + r).encode() * 8 for r in self.realms}, + subjects_by_key_id={r: m.subject for r, m in self.manifests.items()}, + environments_by_key_id={r: r for r in self.realms}) + self.identity = DataPlaneIdentityService(self.config, ConsumerManifestRegistry(tuple(self.manifests.values()))) + + def token(self, realm, **kwargs): + defaults = dict(environment=realm, key_id=realm, secret=self.config.keys_by_id[realm]) + defaults.update(kwargs) + return make_token(self.manifests[realm].subject, **defaults) + + def test_each_mode_authenticates_without_changing_storage_realm(self): + for realm, manifest in self.manifests.items(): + with self.subTest(realm=realm): + access = self.identity.authenticate(self.token(realm), consumer_id=manifest.consumer_id) + self.assertEqual(access.principal.environment, realm) + self.assertEqual(access.manifest, manifest) + self.assertEqual(self.config.environment, "paper") + + def test_key_cannot_assert_another_realm_or_consumer(self): + for realm in self.realms: + for other in set(self.realms) - {realm}: + with self.subTest(realm=realm, other=other): + with self.assertRaises(DataPlaneAccessError): + self.identity.authenticate(self.token(realm, environment=other), consumer_id="ts." + other) + with self.assertRaises(DataPlaneAccessError): + self.identity.authenticate(self.token(realm), consumer_id="ts." + other) + with self.assertRaises(DataPlaneAccessError): + self.identity.authenticate(self.token(realm, key_id=other), consumer_id="ts." + realm) + + def test_subject_and_revision_stay_bound(self): + for realm in self.realms: + with self.assertRaises(DataPlaneAccessError): + self.identity.authenticate(self.token(realm, manifest_revision=2), consumer_id="ts." + realm) + forged = make_token("spiffe://qdl/" + realm + "/other", environment=realm, + key_id=realm, secret=self.config.keys_by_id[realm]) + with self.assertRaises(DataPlaneAccessError): + self.identity.authenticate(forged, consumer_id="ts." + realm) + + def test_unconfigured_legacy_single_realm_stays_closed(self): + legacy = DataPlaneIdentityService(replace(self.config, environments_by_key_id=None), + ConsumerManifestRegistry(tuple(self.manifests.values()))) + legacy.authenticate(self.token("paper"), consumer_id="ts.paper") + for realm in ("sandbox", "live"): + with self.assertRaises(DataPlaneAccessError): + legacy.authenticate(self.token(realm), consumer_id="ts." + realm) + + def test_mapping_is_exact_and_has_no_wildcard(self): + for value in ({}, {"live": "live"}, {r: "*" for r in self.realms}, [], "live"): + with self.subTest(value=value), self.assertRaises(ValueError): + replace(self.config, environments_by_key_id=value) + with self.assertRaises(ValueError): + replace(self.config, subjects_by_key_id=None) + + def test_live_and_sandbox_do_not_accept_paper_only_contract(self): + for realm in ("sandbox", "live"): + with self.assertRaises(ValueError): + replace(self.manifests[realm], execution_dependency="PAPER_ONLY") + + +class ConsumerRealmCompilerTests(unittest.TestCase): + def test_stable_scopes_preserved_for_both_additional_modes(self): + from pathlib import Path + import yaml + from scripts.compile_consumer_realms import compile_realm + root = Path(__file__).resolve().parents[1] + for name in ("trading-system-paper", "alpha-binance-paper", "alpha-okx-paper"): + payload = yaml.safe_load((root / "consumers/stable" / (name + ".yaml")).read_text()) + original = ConsumerManifestLoader.from_mapping(payload) + for realm in ("sandbox", "live"): + result = ConsumerManifestLoader.from_mapping(compile_realm(payload, realm)) + self.assertEqual(result.requirements, tuple(r for r in original.requirements + if r.source_policy_id in {"crypto_primary_v2", "crypto_liquid_v2"})) + self.assertFalse(any(r.source_policy_id == "vn_primary_v2" for r in result.requirements)) + if name == "trading-system-paper": + self.assertEqual(len(result.requirements), 60) + self.assertEqual(result.quotas, original.quotas) + self.assertEqual(result.allowed_permissions, original.allowed_permissions) + self.assertEqual(result.allowed_purposes, original.allowed_purposes) + self.assertNotEqual(result.subject, original.subject) + self.assertNotEqual(result.consumer_id, original.consumer_id) + self.assertEqual(result.environment, realm) + self.assertEqual(payload["metadata"]["environment"], "paper") + self.assertEqual(payload["spec"]["execution_dependency"], original.execution_dependency) + self.assertEqual(result.execution_dependency, + "ALLOWED" if original.execution_dependency == "PAPER_ONLY" else "FORBIDDEN") + + def test_other_consumer_is_not_implicitly_promoted(self): + from scripts.compile_consumer_realms import compile_realm + payload = manifest_mapping(consumer_id="unapproved.paper.stable", + subject="spiffe://qdl/paper/unapproved", instrument_uid="instrument-test") + for policy in ("FORBIDDEN", "PAPER_ONLY", "ALLOWED"): + payload["spec"]["execution_dependency"] = policy + with self.assertRaises(ValueError): + compile_realm(payload, "live") diff --git a/upgrade/DATA_LAYER_V2_KAFKA_NATIVE_ARCHITECTURE_REVIEW.md b/upgrade/DATA_LAYER_V2_KAFKA_NATIVE_ARCHITECTURE_REVIEW.md index 67f48c2c..9e23e2d9 100644 --- a/upgrade/DATA_LAYER_V2_KAFKA_NATIVE_ARCHITECTURE_REVIEW.md +++ b/upgrade/DATA_LAYER_V2_KAFKA_NATIVE_ARCHITECTURE_REVIEW.md @@ -2996,3 +2996,25 @@ fault tests và một final acceptance; không bỏ tests hoặc nới SLA cho k Trạng thái thực thi ban đầu của cả năm phase là NOT STARTED. Source ở thời điểm soạn: `74337e71ce8ce7611ca424d33f7a0ab29535733a`; facts runtime trong mục 0-17 là lịch sử, không có health/latency certification mới từ việc soạn guide. + +### Post-release consumer realms (owner approved 2026-09-27) + +`paper`, `sandbox`, and `live` are workload authorization realms, not different +copies of public mainnet market data. The existing `paper` cache/product/cursor +namespace is retained as the physical market-data realm for compatibility; +it does not certify an execution account mode. Every additional identity has a +separate TLS client identity and JWT signing key. Explicit key-to-realm and +key-to-subject maps must cover the entire keyring. A token cannot select its own +realm independently of its pinned key. Consumer ID, manifest revision, venue/feed +entitlements, purposes, quotas and consumer-bound signed cursor remain enforced. +Old single-realm configuration keeps its existing rejection behavior. + +No extra network authorization hop or service is introduced. Query loads a sealed +configuration on startup; Rust Stream atomically reloads its authority and revokes +an admitted stream if its key realm changes. Configuration is public-key material +only; all private material stays in protected runtime directories outside Git. +`compile_consumer_realms.py` maps only the three approved TS/Binance-alpha/OKX-alpha +scopes. TS PAPER_ONLY becomes ALLOWED for its explicitly approved sandbox/live +read identity. Alpha FORBIDDEN direct execution dependency stays FORBIDDEN: alpha +uses data to construct intent, while TS Risk remains the admission authority. +Read activation never enables orders, changes broker mode, or starts an alpha. From fba88634962d9ba22cc1a056e10b692e4601dc7e Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Sun, 27 Sep 2026 17:09:34 +0000 Subject: [PATCH 04/17] Materialize verified recent bars without certifying overlapping native history --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 39 ++++ qdl/adapters/binance/bar_edge.py | 54 +++-- qdl/adapters/intervals.py | 24 +++ qdl/runtime/kn_query_backend.py | 8 +- qdl/runtime/stable_bar_edge.py | 72 ++++++- tests/test_binance_history_overlap.py | 245 ++++++++++++++++++++++ tests/test_kn_history_fill.py | 102 +++++++++ tests/test_kn_query_backend.py | 23 ++ 8 files changed, 546 insertions(+), 21 deletions(-) create mode 100644 tests/test_binance_history_overlap.py diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index 13f6d8f1..17b00295 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -60349,3 +60349,42 @@ wrong key/realm/consumer/revision, unknown mappings, preserved quotas/purposes, no implicit other-consumer promotion, and FORBIDDEN alpha authority preservation. Ruff targeted security/compiler/tests PASS with --isolated (repository Poetry requires-python caret is rejected by installed Ruff TOML parser otherwise). + +#### Native history and full transport measurement checkpoint + +Native Binance3d overlap confirmed by admitted real provider calls across five +symbols, original response bytes/hashes retained in native3d-audit-20260927T163746Z. +Implemented typed validated recent suffix outcome: Query larger requested windows +remain PARTIAL, edge may materialize only independently verified recent final bars. +No rewriting timestamps, no silently successful short history. Old overlap retries +are daily, sink retry reuses bounded captured suffix, final-bar watermark remains +separate from failed deep-history coverage. Tests114PASS/2explicit skips; isolated +Redis41/41PASS. Receipt native3d-suffix-20260927T170112Z/handoff.json records commands, +source hashes and cleanup. Five further real provider reads preserve original bytes. + +Full capture transport acceptance now includes native Rust gRPC with ephemeral +mTLS, production SDK validation and fsynced cursor ACK, TS projection and isolated +Redis. At2000event/s:110860/110860 across10products in55.710s,0duplicates/reconnects/ +cursorerrors, exact per-product canonical digest order and finalACK; no OOM. +This is capture replay under explicit projection-time clock, not current-price +eligibility nor a Kafka-ingestion capacity test. Constant original provider bytes +are reused20times with test-only monotonic log offsets. No production market writes. + +4000event/s debug attempt FAILED, followed once by optimized release comparison: +release offered110860 in27.714s,max producerlateness5.24ms,hub sampledlag17, +3overflows,client31251received/30991applied before harness stopped on reconnect +superseding an unACKed batch. No OOM/decode/reader/cursorerrors. End queue0 after +abort is NOT recovery success. Do not certify4000fullchain or edit away failure. +Changing test to discard/replay unACKed batch was rejected by automatic review; +no such edit executed. Existing fail evidence is retained. User-requested4000 +capacity question therefore remains negative for this current single consumer. + +New identity manifests prepared, NOT active: TS60 and alpha375 each for sandbox +and live. CA private key never persisted, independent workload TLS/JWT keys, +additive client trust copies only. Scope: existing Query1/2 and Stream1/2; image/ +config rollback recorded before action. No broker ACL, canonical realm, Kafka, +cache, V1, TSorder or alpha mutation. BAR-edge rollout follows Query coverageguard, +normal verified five-symbol3d bootstrap writes only; no offset reset/state deletion. +After suffix publication, do not rollback Query's PARTIAL guard: auth rollback can +restore old paper config on the fixed image. Old query image remains emergency +rollback with the pre-existing3d coverage limitation explicitly not certified. diff --git a/qdl/adapters/binance/bar_edge.py b/qdl/adapters/binance/bar_edge.py index a68513a6..adafdf1f 100644 --- a/qdl/adapters/binance/bar_edge.py +++ b/qdl/adapters/binance/bar_edge.py @@ -7,7 +7,10 @@ from typing import Callable from app.providers.binance.rest import BinanceProviderError, fetch_klines -from qdl.adapters.intervals import BarHistoryGapError, canonical_interval_ms, latest_closed_boundary_ms +from qdl.adapters.intervals import ( + BarHistoryGapError, BarHistoryOverlapError, canonical_interval_ms, + is_valid_bar_open_ms, latest_closed_boundary_ms, +) from qdl.provider.v1 import raw_provider_pb2 from qdl.raw.capture import capture_exact_frame @@ -236,6 +239,43 @@ def fetch_closed_bar_history_raw_envelopes( observed_ms=observed_ms, interval_ms=interval_ms, ) + if len(closed) > page_limit or any(int(row[0]) > page_end_ms for row in closed): + raise RuntimeError("Binance closed-bar history page exceeds requested end boundary") + for row in closed: + open_time = int(row[0]) + previous = selected_by_open.get(open_time) + if previous is not None and previous != row: + raise RuntimeError("Binance history pages conflict for one open time") + selected_by_open[open_time] = row + selected = tuple(selected_by_open[key] for key in sorted(selected_by_open)) + # Walk newest first: only the suffix after the newest discontinuity can + # be certified, including when that discontinuity crosses REST pages. + for index in range(len(selected) - 1, 0, -1): + previous, current = int(selected[index - 1][0]), int(selected[index][0]) + if current - previous == interval_ms: + continue + error = BarHistoryGapError("Binance", binding.native_symbol, binding.interval, + previous, current) + suffix = selected[index:] + latest_open = latest_closed_boundary_ms( + binding.interval, observed_ms, provider="BINANCE" + ) - interval_ms + if (error.kind == "OVERLAPPING_PROVIDER_WINDOWS" and binding.interval == "3d" + and int(suffix[-1][0]) == latest_open + and all(is_valid_bar_open_ms(binding.interval, int(row[0]), provider="BINANCE") + for row in suffix)): + received_at_ns = time.time_ns() + envelopes = tuple(_capture_row( + binding, row, origin="BACKFILLED", received_at_ns=received_at_ns + offset, + test_provenance=test_provenance, + ) for offset, row in enumerate(suffix)) + raise BarHistoryOverlapError( + "Binance", binding.native_symbol, binding.interval, previous, current, + requested_rows=limit, recent_envelopes=envelopes, + ) + raise error + if any(int(row[6]) > page_end_ms for row in closed): + raise RuntimeError("Binance closed-bar history page exceeds requested end boundary") exhausted = False if len(closed) != page_limit: if not (allow_short and len(closed) < page_limit and _provider_exhausted_before( @@ -247,18 +287,6 @@ def fetch_closed_bar_history_raw_envelopes( f"requested={page_limit} observed={len(closed)}" ) exhausted = True - if any(int(row[6]) > page_end_ms for row in closed): - raise RuntimeError( - "Binance closed-bar history page exceeds requested end boundary" - ) - for row in closed: - open_time = int(row[0]) - previous = selected_by_open.get(open_time) - if previous is not None and previous != row: - raise RuntimeError( - "Binance history pages conflict for one open time" - ) - selected_by_open[open_time] = row if exhausted or not closed: break earliest_open_ms = min(int(row[0]) for row in closed) diff --git a/qdl/adapters/intervals.py b/qdl/adapters/intervals.py index ad7ff6a1..49d1e2e6 100644 --- a/qdl/adapters/intervals.py +++ b/qdl/adapters/intervals.py @@ -23,6 +23,30 @@ def __init__(self, provider: str, symbol: str, interval: str, previous: int, cur super().__init__(f"{provider} closed-bar history discontinuity kind={self.kind} " f"symbol={symbol} interval={interval} previous_ms={previous} current_ms={current}") + def retry_delay_seconds(self, attempt: int) -> int: + # Overlapping native windows cannot be repaired by fetching a missing + # bar. Recheck for provider corrections daily, retaining failed coverage. + if self.kind == "OVERLAPPING_PROVIDER_WINDOWS": + return 86_400 + return min(2 ** min(attempt, 6), 30) + + +class BarHistoryOverlapError(BarHistoryGapError): + """Incomplete requested history with an independently validated recent suffix. + + Only history materialization may use these envelopes. Request fetchers still + raise: this is never a successful short response or proof of a listing floor. + """ + + def __init__(self, provider: str, symbol: str, interval: str, previous: int, + current: int, *, requested_rows: int, recent_envelopes: tuple): + super().__init__(provider, symbol, interval, previous, current) + if self.kind != "OVERLAPPING_PROVIDER_WINDOWS" or not recent_envelopes: + raise ValueError("history overlap outcome requires a nonempty recent suffix") + self.requested_rows = requested_rows + self.recent_envelopes = recent_envelopes + self.older_prefix_status = "UNCERTIFIED_PROVIDER_OVERLAP" + _UNIT_MS = { "s": 1_000, diff --git a/qdl/runtime/kn_query_backend.py b/qdl/runtime/kn_query_backend.py index ad6b27d9..25fafb8a 100644 --- a/qdl/runtime/kn_query_backend.py +++ b/qdl/runtime/kn_query_backend.py @@ -47,7 +47,7 @@ from qdl.adapters.intervals import canonical_interval_ms from qdl.common.v1 import common_pb2 from qdl.marketdata.v2 import market_data_pb2 -from qdl.query import DataRequirement, FeedType, GapRecord, HistoryResult, MarketDataItem, RecoveryPolicy +from qdl.query import CoverageStatus, DataRequirement, FeedType, GapRecord, HistoryResult, MarketDataItem, RecoveryPolicy from qdl.query.contracts import CanonicalErrorCode, QueryProblem from qdl.query.results import NON_REPLAYABLE_STREAM_CURSOR, QueryBackendError, GapScanResult from qdl.query.row_cache import BoundedRowCache @@ -368,6 +368,12 @@ def history_with_envelopes( ) if result is None: return None, () + # Native Binance 3d has overlapping historical grids. A retained recent + # suffix does not prove listing exhaustion or certify an older prefix. + if (binding.feed is FeedType.BAR and binding.interval == "3d" + and binding.instrument.identity.venue == "BINANCE" + and start_ns is None and len(result.items) < requested): + result = replace(result, coverage=CoverageStatus.PARTIAL) selected = records if start_ns is not None: selected = tuple( diff --git a/qdl/runtime/stable_bar_edge.py b/qdl/runtime/stable_bar_edge.py index 37abddf5..43a84102 100644 --- a/qdl/runtime/stable_bar_edge.py +++ b/qdl/runtime/stable_bar_edge.py @@ -14,6 +14,7 @@ from qdl.adapters.intervals import ( BarHistoryGapError, + BarHistoryOverlapError, canonical_interval_ms, is_valid_bar_open_ms, latest_closed_boundary_ms, @@ -347,6 +348,8 @@ def __init__( self.history_demand = dict(history_demand) if history_demand is not None else None self.history_gate = history_gate self._history_short: dict[str, tuple[int, int]] = {} + self._history_suspended: dict[str, dict] = {} + self._history_suffix_pending: dict[str, BarHistoryOverlapError] = {} self._history_gate_closed_at: float | None = None # KN-4 D47-4: the history/live join. History holds bars closed before # ``history_end_ms`` (the live log's start, e.g. the mirror start); @@ -487,6 +490,7 @@ def _state_payload(self) -> dict: "connection_generation": self.connection_generation, "last_open_ms": { key: self._last_open_ms[key] for key in sorted(self._last_open_ms) + if key not in getattr(self, "_history_suspended", {}) }, } @@ -825,7 +829,9 @@ def _apply_history_repair_plan( self._assert_canonical_cache_identity() if advance_watermark: # Publish progress: the live watermark may move on the Kafka ACK. - self._last_open_ms[plan.source.binding_id] = max(plan.expected_opens) + self._last_open_ms[plan.source.binding_id] = max( + max(plan.expected_opens), self._last_open_ms.get(plan.source.binding_id, 0) + ) self._persist_state() self._record_serving(plan.source.binding_id, plan.expected_opens, published=bool(current.missing_envelopes)) @@ -1062,6 +1068,8 @@ def _rebase_if_canonical_cache_generation_changed(self) -> bool: previous_cache_id = self.canonical_cache_id self.canonical_cache_id = observed_cache_id self._last_open_ms.clear() + for name in ("_history_suspended", "_history_suffix_pending", "_history_retry"): + getattr(self, name, {}).clear() self._native_recovery_pending.clear() self._native_recovery_verified_open.clear() self._native_gap_first_seen_at.clear() @@ -1315,6 +1323,8 @@ def _rebase_changed_products(self) -> tuple[str, ...]: if previous is not None and generation != previous: rebased.append(binding_id) self._last_open_ms.pop(binding_id, None) + for name in ("_history_suspended", "_history_suffix_pending", "_history_retry"): + getattr(self, name, {}).pop(binding_id, None) for store in (self._native_recovery_pending, self._native_recovery_verified_open, self._native_gap_first_seen_at, self._native_recovery_visible_after, self._retry_attempts, self._next_retry_at, self._serving_pending): @@ -1379,6 +1389,7 @@ def serving_status(self) -> dict[str, int]: "served": len(self._served), "pending": len(self._serving_pending), "overdue": len(self._serve_overdue), "short": len(getattr(self, "_history_short", {})), "history_failed": len(getattr(self, "_history_retry", {})), + "history_suspended": len(getattr(self, "_history_suspended", {})), "gate_closed": int(getattr(self, "_history_gate_closed_at", None) is not None)} def bootstrap_history(self, *, max_bindings: int | None = None) -> int: @@ -1401,8 +1412,13 @@ def bootstrap_history(self, *, max_bindings: int | None = None) -> int: kn_mode = getattr(self, "bar_readback", None) is not None failures = getattr(self, "_history_retry", {}) self._history_retry = failures + suspended = getattr(self, "_history_suspended", {}) + self._history_suspended = suspended + pending = getattr(self, "_history_suffix_pending", {}) + self._history_suffix_pending = pending for source, acquisition in self.history_bindings + self.history_okx_bindings: - if source.binding_id in self._last_open_ms or source.binding_id in self._history_short_empty(): + if ((source.binding_id in self._last_open_ms and source.binding_id not in suspended) + or source.binding_id in self._history_short_empty()): continue if kn_mode and not self._history_gate_open(): return published # resumed on a later turn; live bars keep flowing @@ -1414,15 +1430,53 @@ def bootstrap_history(self, *, max_bindings: int | None = None) -> int: attempted += 1 bootstrap_rows = self._bootstrap_rows_for(source) try: + if source.binding_id in pending: + raise pending[source.binding_id] values = self._fetch_history(source, acquisition, rows=bootstrap_rows, observed_ms=observed_ms, allow_short=kn_mode) except BarHistoryGapError as error: if not kn_mode: raise attempt += 1 - failures[source.binding_id] = (attempt, self.clock() + min(2 ** min(attempt, 6), 30)) - logger.error("stable BAR provider history refused binding=%s attempt=%s error=%s", - source.binding_id, attempt, error) + retry_delay = error.retry_delay_seconds(attempt) + failures[source.binding_id] = (attempt, self.clock() + retry_delay) + logger.error("stable BAR provider history refused binding=%s attempt=%s retry_in_seconds=%s error=%s", + source.binding_id, attempt, retry_delay, error) + if isinstance(error, BarHistoryOverlapError): + values = error.recent_envelopes + opens = tuple(self._open_time_ms(acquisition, item) for item in values) + # Keep failed deep coverage separate from a live watermark. + # The legacy checkpoint excludes suspended bindings, so a + # restart rechecks them rather than claiming full bootstrap. + suspended[source.binding_id] = { + "status": error.older_prefix_status, "requested_rows": error.requested_rows, + "previous_ms": error.previous, "current_ms": error.current, + "recent_from_ms": min(opens), "recent_rows": len(values), + } + # Retain only the outcome, not traceback frames from each + # failed sink retry (which would accumulate indefinitely). + pending[source.binding_id] = BarHistoryOverlapError( + error.provider, error.symbol, error.interval, error.previous, error.current, + requested_rows=error.requested_rows, recent_envelopes=values, + ) + try: + published += self._publish_history( + source, acquisition, values, expected_rows=len(values), + ) + except Exception: + # Retry publication of these same bytes, not provider + # pagination, when the durable sink temporarily fails. + failures[source.binding_id] = (attempt, self.clock() + 30) + raise + pending.pop(source.binding_id, None) + logger.warning("stable BAR recent suffix materialized binding=%s rows=%s " + "first_open_ms=%s older_prefix=%s requested_rows=%s", + source.binding_id, len(values), min(opens), + error.older_prefix_status, error.requested_rows) + continue + if source.binding_id in suspended and not values: + failures[source.binding_id] = (attempt + 1, self.clock() + 86_400) + logger.error("stable BAR suspended history recheck empty binding=%s", source.binding_id) continue failures.pop(source.binding_id, None) if len(values) < bootstrap_rows: @@ -1439,8 +1493,11 @@ def bootstrap_history(self, *, max_bindings: int | None = None) -> int: values, expected_rows=len(values), ) + if source.binding_id in suspended: + suspended.pop(source.binding_id) + self._persist_state() self._history_bootstrapped = ( - set(self._last_open_ms) | self._history_short_empty() == set(self._binding_ids) + (set(self._last_open_ms) - set(suspended)) | self._history_short_empty() == set(self._binding_ids) ) if not self._history_bootstrapped: if kn_mode and failures: @@ -1935,7 +1992,8 @@ def _history_loop_delay(self, now: float) -> float | None: or getattr(self, "_history_bootstrapped", False)): return None retries = getattr(self, "_history_retry", {}) - missing = set(self._binding_ids) - set(self._last_open_ms) - self._history_short_empty() + complete = set(self._last_open_ms) - set(getattr(self, "_history_suspended", {})) + missing = set(self._binding_ids) - complete - self._history_short_empty() if not missing: return None return max(.1, min(max(0., retries.get(key, (0, now))[1] - now) for key in missing)) diff --git a/tests/test_binance_history_overlap.py b/tests/test_binance_history_overlap.py new file mode 100644 index 00000000..ec836c70 --- /dev/null +++ b/tests/test_binance_history_overlap.py @@ -0,0 +1,245 @@ +"""Test-only synthetic OHLCV; provider-observed 3d transition timestamps.""" +from dataclasses import dataclass, replace +import json +from types import SimpleNamespace +import unittest +from unittest.mock import Mock, patch + +from qdl.adapters.binance.bar_edge import fetch_closed_bar_history_raw_envelopes +from qdl.adapters.intervals import BarHistoryGapError, BarHistoryOverlapError +from qdl.query import ConsumerGrade, CoverageStatus, DataRequirement, FeedType, HistoryResult +from qdl.query.contracts import CanonicalErrorCode, evaluate_requirement +from qdl.runtime.kn_query_backend import KnMarketCacheQueryBackend +from qdl.runtime.stable_bar_edge import StableBinanceBarEdge +import tests.test_kn_history_fill as fixtures + +DURATION = 259_200_000 +START = 1_692_144_000_000 + + +def row(open_ms): + value = fixtures.ShortProviderHistoryTests._row(open_ms) + value[6] = open_ms + DURATION - 1 + return value + + +def binding(): + return replace(fixtures.ShortProviderHistoryTests()._binding(), + native_symbol="ETHUSDT", interval="3d") + + +def provider(rows): + def fetch(_symbol, *, end_time, limit, **kwargs): + return {"data": [value for value in rows if value[0] <= end_time][-limit:]} + return Mock(side_effect=fetch) + + +def overlapping_rows(): + return [row(START - 86_400_000 * 2)] + [row(START + i * DURATION) for i in range(3)] + + +def fetch(fetcher, **kwargs): + return fetch_closed_bar_history_raw_envelopes( + binding(), now_ms=START + 3 * DURATION + 1, attempts=1, + fetcher=fetcher, test_provenance=True, **kwargs, + ) + + +class SuffixOutcomeTests(unittest.TestCase): + def test_same_page_suffix_raises_and_preserves_exact_native_rows(self): + rows = overlapping_rows() + call = provider(rows) + with self.assertRaises(BarHistoryOverlapError) as caught: + fetch(call, limit=10000, allow_short=True) + error = caught.exception + self.assertEqual(error.requested_rows, 10000) + self.assertEqual(error.older_prefix_status, "UNCERTIFIED_PROVIDER_OVERLAP") + self.assertEqual([json.loads(v.raw_frame_bytes)["row"] for v in error.recent_envelopes], rows[1:]) + self.assertEqual(call.call_count, 1, "do not fetch an older page or probe exhaustion after overlap") + self.assertEqual(len(fetch(provider(rows), limit=3)), 3) + + def test_cross_page_overlap_retains_all_verified_newer_rows(self): + call = provider(overlapping_rows()) + with patch("qdl.adapters.binance.bar_edge._HISTORY_PAGE_ROWS", 3): + with self.assertRaises(BarHistoryOverlapError) as caught: + fetch(call, limit=10000, allow_short=True) + self.assertEqual(len(caught.exception.recent_envelopes), 3) + self.assertEqual(call.call_count, 2) + + def test_only_newest_contiguous_segment_can_be_materialized(self): + rows = [row(START - 86_400_000 * 4), row(START - 86_400_000 * 2)] + overlapping_rows()[1:] + with self.assertRaises(BarHistoryOverlapError) as caught: + fetch(provider(rows), limit=5) + self.assertEqual(caught.exception.previous, START - 86_400_000 * 2) + self.assertEqual(len(caught.exception.recent_envelopes), 3) + + def test_off_grid_stale_and_missing_suffix_never_offer_materialization(self): + for rows in ( + [row(value[0] - 1) for value in overlapping_rows()], + overlapping_rows()[:-1], + [row(START - DURATION * 2), row(START), row(START + 2 * DURATION)], + ): + with self.subTest(opens=[r[0] for r in rows]): + with self.assertRaises(BarHistoryGapError) as caught: + fetch(provider(rows), limit=len(rows)) + self.assertNotIsInstance(caught.exception, BarHistoryOverlapError) + + def test_invalid_close_rejected_before_any_suffix_is_offered(self): + rows = overlapping_rows() + rows[-1][6] -= 1 + with self.assertRaises(ValueError): + fetch(provider(rows), limit=4) + + +class SuffixMaterializationTests(unittest.TestCase): + def edge(self): + fixture = fixtures.EdgeFillTests() + edge = fixture._edge(demand=None, gate=None) + source = edge.history_bindings[0][0] + source.interval = "3d" + acquisition = SimpleNamespace(runtime="BINANCE") + edge.history_bindings = ((source, acquisition),) + edge._history_short = {"b": (1, 0), "c": (1, 0)} + edge._settled_observed_ms = lambda: START + 3 * DURATION + 1 + edge._assert_repair_writer_current = lambda: None + edge._assert_canonical_cache_identity = lambda: None + edge._persist_state = Mock() + edge._record_serving = Mock() + edge._publish_history = StableBinanceBarEdge._publish_history.__get__(edge) + durable = {START + 2 * DURATION} + edge._durable_final_bar_opens = lambda _source, opens: frozenset(durable & opens) + emitted = [] + + def publish(values): + values = tuple(values) + emitted.extend(values) + durable.update(json.loads(value.raw_frame_bytes)["row"][0] for value in values) + return tuple(object() for _ in values) + + edge.publisher = SimpleNamespace(publish_many=Mock(side_effect=publish)) + call = provider(overlapping_rows()) + edge._fetch_history = lambda _source, _acquisition, rows, **kwargs: fetch( + call, limit=rows, allow_short=True) + return edge, call, durable, emitted + + def test_bootstrap_materializes_recent_three_but_never_completes_deep_history(self): + edge, call, durable, emitted = self.edge() + self.assertEqual(edge.bootstrap_history(), 2) + self.assertEqual(durable, {START + i * DURATION for i in range(3)}) + self.assertEqual(len(emitted), 2, "existing newest bar is not republished") + self.assertTrue(edge._live_history_ready("a")) + self.assertEqual(edge._last_open_ms["a"], START + 2 * DURATION) + self.assertFalse(edge._history_bootstrapped) + self.assertNotIn("a", edge._history_short) + self.assertEqual(edge._history_suspended["a"]["status"], "UNCERTIFIED_PROVIDER_OVERLAP") + for now in (31, 3601, 86400): + edge.clock = lambda: now + edge.bootstrap_history() + self.assertEqual(call.call_count, 1) + self.assertEqual(edge.publisher.publish_many.call_count, 1) + self.assertEqual(edge._history_loop_delay(86400), 1) + edge.clock = lambda: 86401 + self.assertEqual(edge.bootstrap_history(), 0) + self.assertEqual(call.call_count, 2) + self.assertFalse(edge._history_bootstrapped) + edge.canonical_cache_id = "test-cache" + edge.connection_generation = 1 + edge._state_identity_payload = lambda **kwargs: {"schema": kwargs["schema"]} + self.assertEqual(edge._state_payload()["last_open_ms"], {}, "restart cannot certify suffix as full history") + + def test_next_final_bar_advances_while_deep_history_remains_suspended(self): + edge, call, durable, emitted = self.edge() + edge.bootstrap_history() + edge.bindings = edge.history_bindings + edge.okx_bindings = () + edge._rest_fallback_active = True + edge.max_catchup_rows = 10000 + edge._binding_is_due = lambda *args, **kwargs: True + latest = fetch_closed_bar_history_raw_envelopes( + binding(), now_ms=START + 4 * DURATION + 1, limit=1, attempts=1, + fetcher=provider([row(START + 3 * DURATION)]), test_provenance=True)[0] + edge._fetch_latest = Mock(return_value=latest) + edge._settled_observed_ms = lambda: START + 4 * DURATION + 1 + self.assertEqual(edge.run_cycle(), 1) + self.assertEqual(edge._last_open_ms["a"], START + 3 * DURATION) + self.assertIn(START + 3 * DURATION, durable) + self.assertFalse(edge._history_bootstrapped) + self.assertIn("a", edge._history_retry) + self.assertIn("a", edge._history_suspended) + self.assertEqual(call.call_count, 1) + + def test_empty_recheck_cannot_certify_previously_rejected_history(self): + edge, call, durable, emitted = self.edge() + edge.bootstrap_history() + edge.clock = lambda: 86401 + edge._fetch_history = lambda *args, **kwargs: () + self.assertEqual(edge.bootstrap_history(), 0) + self.assertFalse(edge._history_bootstrapped) + self.assertIn("a", edge._history_suspended) + self.assertIn("a", edge._history_retry) + self.assertNotIn("a", edge._history_short) + + def test_sink_failure_reuses_pending_provider_bytes_without_refetch(self): + edge, call, durable, emitted = self.edge() + publish = edge.publisher.publish_many.side_effect + edge.publisher.publish_many.side_effect = OSError("test-only sink failure") + with self.assertRaises(OSError): + edge.bootstrap_history() + self.assertNotIn("a", edge._last_open_ms) + self.assertFalse(edge._history_bootstrapped) + self.assertEqual(call.call_count, 1) + edge.clock = lambda: 31 + edge.publisher.publish_many.side_effect = publish + self.assertEqual(edge.bootstrap_history(), 2) + self.assertEqual(call.call_count, 1) + self.assertEqual(len(durable), 3) + self.assertFalse(edge._history_suffix_pending) + + def test_verified_provider_correction_can_complete_without_rewinding_live(self): + edge, call, durable, emitted = self.edge() + edge.bootstrap_history() + edge.clock = lambda: 86401 + edge._last_open_ms["a"] = START + 3 * DURATION + edge._fetch_history = lambda *args, **kwargs: fetch(provider(overlapping_rows()), limit=3) + edge.bootstrap_history() + self.assertTrue(edge._history_bootstrapped) + self.assertFalse(edge._history_suspended) + self.assertFalse(edge._history_retry) + self.assertEqual(edge._last_open_ms["a"], START + 3 * DURATION) + + +@dataclass(frozen=True) +class Item: + snapshot_id: str = "test-only" + + +class SuffixQueryCoverageTests(unittest.TestCase): + def result(self, requested, *, venue="BINANCE", interval="3d", start_ns=None): + backend = KnMarketCacheQueryBackend.__new__(KnMarketCacheQueryBackend) + source = SimpleNamespace(feed=FeedType.BAR, interval=interval, + instrument=SimpleNamespace(identity=SimpleNamespace(venue=venue))) + backend.catalog = SimpleNamespace(binding_for=lambda _: source) + backend._requested_window = lambda _: (requested, start_ns, 4, None) + records = tuple(SimpleNamespace(envelope=SimpleNamespace(bar=SimpleNamespace(open_time_ns=i))) for i in (1, 2, 3)) + view = SimpleNamespace(boundary=SimpleNamespace(offset=3)) + backend._history_view = lambda *args: (view, records) + backend._history_from_records = lambda *args, **kwargs: HistoryResult( + items=tuple(Item() for _ in records), coverage=CoverageStatus.FULL, + snapshot_id="test-only", stream_cursor="test-only", watermark_offset=3, data_as_of_ns=3) + with patch("qdl.runtime.kn_query_backend.view_snapshot_id", return_value="test-only"), patch( + "qdl.runtime.kn_query_backend.source_placeholder", return_value="test-only"): + return backend.history_with_envelopes(None)[0] + + def test_exact_recent_request_full_but_uncertified_older_prefix_is_partial(self): + self.assertEqual(self.result(3).coverage, CoverageStatus.FULL) + result = self.result(10000) + self.assertEqual(result.coverage, CoverageStatus.PARTIAL) + requirement = DataRequirement(instrument_uid="test-only", feed=FeedType.BAR, + interval="3d", consumer_grade=ConsumerGrade.ALPHA, + source_policy_id="test-only", warmup_limit=10000) + problem = evaluate_requirement(requirement, coverage=result.coverage, entitled=True, + available=True, fresh=True, authoritative=True, gap_open=False) + self.assertEqual(problem.code, CanonicalErrorCode.PARTIAL_RESULT) + self.assertEqual(self.result(10000, venue="OKX").coverage, CoverageStatus.FULL) + self.assertEqual(self.result(10000, interval="1m").coverage, CoverageStatus.FULL) + self.assertEqual(self.result(10000, start_ns=1).coverage, CoverageStatus.FULL) diff --git a/tests/test_kn_history_fill.py b/tests/test_kn_history_fill.py index 624320ba..f025d1a4 100644 --- a/tests/test_kn_history_fill.py +++ b/tests/test_kn_history_fill.py @@ -31,6 +31,55 @@ def _bundle(): return {"catalog": {"bindings": bindings}, "manifests": manifests} +class BinanceNativeThreeDayTests(unittest.TestCase): + """Synthetic OHLCV with provider-observed timestamps; never provider replay.""" + + def test_recent_exact_window_and_overlap_never_become_truncated_success(self): + from dataclasses import replace + import json + from unittest.mock import patch + from qdl.adapters.binance.bar_edge import fetch_closed_bar_history_raw_envelopes + from qdl.adapters.intervals import BarHistoryGapError + + duration = 259200000 + for symbol, previous, current in ( + ("BTCUSDT", 1569110400000, 1569283200000), + ("ETHUSDT", 1691971200000, 1692144000000), + ("BNBUSDT", 1691971200000, 1692144000000), + ("SOLUSDT", 1692057600000, 1692144000000), + ("DOGEUSDT", 1692057600000, 1692144000000), + ): + binding = replace(ShortProviderHistoryTests()._binding(), + native_symbol=symbol, interval="3d") + rows = [ShortProviderHistoryTests._row(t) for t in + (previous - duration, previous, current, current + duration)] + for row in rows: + row[6] = row[0] + duration - 1 + saved = [list(row) for row in rows] + calls = [] + + def fetch(_symbol, *, limit, end_time, **kwargs): + calls.append((limit, end_time)) + return {"data": [row for row in rows if row[0] <= end_time][-limit:]} + + kwargs = dict(now_ms=current + 3 * duration, attempts=1, + fetcher=fetch, test_provenance=True) + with self.subTest(symbol=symbol, window="recent"): + values = fetch_closed_bar_history_raw_envelopes(binding, limit=2, **kwargs) + self.assertEqual([json.loads(v.raw_frame_bytes)["row"] for v in values], rows[-2:]) + for page_size, limit, allow_short in ((1000, 4, False), (2, 4, False), (1000, 10, True)): + with self.subTest(symbol=symbol, page_size=page_size, allow_short=allow_short): + with patch("qdl.adapters.binance.bar_edge._HISTORY_PAGE_ROWS", page_size): + with self.assertRaises(BarHistoryGapError) as caught: + fetch_closed_bar_history_raw_envelopes( + binding, limit=limit, allow_short=allow_short, **kwargs) + self.assertEqual(caught.exception.kind, "OVERLAPPING_PROVIDER_WINDOWS") + self.assertEqual((caught.exception.previous, caught.exception.current), + (previous, current)) + self.assertEqual(rows, saved) + self.assertLessEqual(len(calls), 6) + + class DemandTests(unittest.TestCase): def test_the_largest_demand_of_a_requiring_manifest_per_bar_binding(self): self.assertEqual(demanded_history_rows(_bundle()), {"b-1m": 10000, "b-1h": 500, "b-nobody": 0}) @@ -231,6 +280,59 @@ def publish(source, _acquisition, values, *, expected_rows): edge._publish_history = publish return edge + def test_overlap_rechecks_daily_without_completing_or_starving_other_history(self): + from qdl.adapters.intervals import BarHistoryGapError + edge = self._edge(demand=None, gate=None) + original = edge._fetch_history + calls = [] + + def refused(source, acquisition, **kwargs): + if source.binding_id == "a": + calls.append(source.binding_id) + raise BarHistoryGapError("Binance", "TEST_ONLY", "3d", + 1691971200000, 1692144000000) + return original(source, acquisition, **kwargs) + + edge._fetch_history = refused + edge.bootstrap_history() + self.assertEqual(edge._history_retry, {"a": (1, 86401.0)}) + self.assertNotIn("a", edge._last_open_ms) + self.assertNotIn("a", edge._history_short) + self.assertFalse(edge._history_bootstrapped) + self.assertEqual(self.published, [("b", 7)]) + edge._rest_fallback_active = False + self.assertEqual(edge._loop_sleep_seconds(1), 60) + for now in (31, 61, 3601, 86400): + edge.clock = lambda: now + edge.bootstrap_history() + self.assertEqual(calls, ["a"]) + edge.clock = lambda: 86401.0 + edge.bootstrap_history() + self.assertEqual(calls, ["a", "a"]) + self.assertEqual(edge._history_retry, {"a": (2, 172801.0)}) + edge.clock = lambda: 172801.0 + edge._fetch_history = original + edge.bootstrap_history() + self.assertEqual(edge._history_retry, {}) + self.assertTrue(edge._history_bootstrapped) + + def test_missing_window_keeps_bounded_transient_retry_and_legacy_raises(self): + from qdl.adapters.intervals import BarHistoryGapError + error = BarHistoryGapError("Binance", "TEST_ONLY", "1m", 60000, 180000) + self.assertEqual([error.retry_delay_seconds(n) for n in (1, 2, 5, 1000)], + [2, 4, 30, 30]) + edge = self._edge(demand=None, gate=None) + + def refused(*args, **kwargs): + raise error + + edge._fetch_history = refused + edge.bootstrap_history(max_bindings=1) + self.assertEqual(edge._history_retry, {"a": (1, 3.0)}) + edge.bar_readback = None + with self.assertRaises(BarHistoryGapError): + edge.bootstrap_history(max_bindings=1) + def test_depth_by_demand_and_short_or_empty_history_are_reported(self): edge = self._edge(demand={"a": 5000, "b": 10_000, "c": 0}, gate=None) edge.bootstrap_history() diff --git a/tests/test_kn_query_backend.py b/tests/test_kn_query_backend.py index ed1b15a3..0edfb082 100644 --- a/tests/test_kn_query_backend.py +++ b/tests/test_kn_query_backend.py @@ -395,6 +395,29 @@ def test_bar_history_matches_the_spool_semantics_at_the_view_boundary(self): self.assertEqual(item.payload, spool.items[-1].payload) self.assertEqual(parse_placeholder(item.cursor), (TOPIC_ID, 3, 4_321)) + def test_binance_3d_suffix_does_not_certify_the_uncaptured_older_prefix(self): + from qdl.query import CoverageStatus + from qdl.query.contracts import evaluate_requirement + + pk, real_bar = next((pk, raw) for pk, raw in golden_records(bar=True) + if pk.endswith("binance-usdm-dogeusdt-bar-3d-primary-v2")) + binding = binding_of(self.catalog, pk, real_bar) + rows = [derived_bar(real_bar, -shift) for shift in range(3)] + self.now_ns = (open_ms_of(real_bar) + canonical_interval_ms("3d")) * 1_000_000 + 5_000_000_000 + self.put_bars(binding, rows) + backend = self.backend() + recent = backend.history(requirement(binding, 3)) + self.assertEqual(len(recent.items), 3) + self.assertEqual(recent.coverage, CoverageStatus.FULL) + full_request = requirement(binding, 10000) + incomplete = backend.history(full_request) + self.assertEqual(len(incomplete.items), 3) + self.assertEqual(incomplete.coverage, CoverageStatus.PARTIAL) + problem = evaluate_requirement(full_request, coverage=incomplete.coverage, + entitled=True, available=True, fresh=True, + authoritative=True, gap_open=False) + self.assertEqual(problem.code, CanonicalErrorCode.PARTIAL_RESULT) + def test_a_time_range_reads_exactly_its_opens(self): payloads = self.history_rows(300) self.put_bars(self.bar_binding, payloads, offsets=[10 + index for index in range(300)], mark=400) From a179bdbff6fa5ab2f8c962de2ab862866deba8f6 Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Sun, 27 Sep 2026 17:26:19 +0000 Subject: [PATCH 05/17] Record three-realm reader acceptance and native burst capacity boundaries --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 52 ++ .../examples/kn_trade_burst_gateway.rs | 545 ++++++++++++++++++ scripts/accept_consumer_realms.py | 130 +++++ scripts/kn_trade_burst_build.sh | 7 + scripts/kn_trade_burst_tls.py | 61 ++ scripts/kn_trade_burst_wire_smoke.py | 96 +++ ...YER_V2_KAFKA_NATIVE_ARCHITECTURE_REVIEW.md | 25 + 7 files changed, 916 insertions(+) create mode 100644 rust/qdl-stream-gateway/examples/kn_trade_burst_gateway.rs create mode 100644 scripts/accept_consumer_realms.py create mode 100644 scripts/kn_trade_burst_build.sh create mode 100644 scripts/kn_trade_burst_tls.py create mode 100644 scripts/kn_trade_burst_wire_smoke.py diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index 17b00295..18d8d326 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -60388,3 +60388,55 @@ normal verified five-symbol3d bootstrap writes only; no offset reset/state delet After suffix publication, do not rollback Query's PARTIAL guard: auth rollback can restore old paper config on the fixed image. Old query image remains emergency rollback with the pre-existing3d coverage limitation explicitly not certified. + +#### Three-realm reader packet (2026-09-27) + +Owner approval covers paper/sandbox/live read authentication, not order activation. +Prepared consumer-realms/reader-candidate.compose.json and reader-rollback.compose.json +outside Git. Query candidate sha256:0ed57643ae40087fe7d35ec04a778b2c8847d520f02aa0394a87b71dcdbeb959; +Stream candidate sha256:65918a656dcb92ab92b687a92195caa5642a185dfdc60c4bd3ebbc702609c09a. +Both source fba8863. Roll exactly query_kn_1/2 and stream_kn_1/2 sequentially. +Rollback Query to 3dd54bfd, Stream to 7fe34806 using prior mounts/config. +Trust changes are additive private copies, not writes to shared stable_tls. +No Kafka, cache, TS, alpha or order changes. BAR writer remains unchanged here. +First packaged unit invocation: six auth tests passed; two compiler tests could +not import development scripts. Rerun with read-only compiler/manifest fixtures; +do not count missing-fixture errors as successful packaged acceptance. + +#### Three-realm runtime acceptance and cleanup + +Status: AUTH_REALM_MAPPING_ACTIVE / EXECUTION_ACCEPTANCE_NOT_FULL. Packaged realm +suite8/8PASS after mounting development-only compiler fixtures. Sequential rolling +completed at17:18:07UTC for exactly two Query and two Stream roles. Query health +healthy; Stream has no Docker healthcheck, verified through real RPC and metrics. +All four restart_count0/OOMfalse; no other service recreated. TS remains paper +identity and V2_PRIMARY, heartbeat60/60sessionREADY,50/60executionREADY,0fallback/ +blocked/error at17:20:51UTC. No alpha started and no order rights activated. + +First auth probe:288typed responses,36negative refusals, but12stream ACK failures +were a harness/SDK packaging mismatch: reader image bundles pre-async-ACK SDK. +No production change required; mounted committed consumer SDK2.0.5 read-only in +the replacement client. Keep first receipt. Replacement:12/12stream receive+ACK, +36/36wrong realm/revision/consumer refused,287/288typed responses. One OKX BTC +MARK_INDEX_PRICE on Query2 refused SOURCE_NON_AUTHORITATIVE at SDK eligibility +validation; four quiet TRADE typed views not execution eligible.283/288usable. +Do not claim full execution acceptance or retry seeking a green observation. +Candidate auth is active; public data quality rules remain unchanged. + +Usable snapshot call-to-validated-result latency (48requests/feed, not p99): +QUOTE median8.27/max22.02ms; MARK median8.36/max12.98ms (47eligible); +BOOK_SNAPSHOT21.14/46.08ms; BOOK_DELTA10.35/67.84ms; BAR13.88/24.68ms; +TRADE8.44/156.75ms (44eligible). These are request latency, not venue event age +or TS Redis commit latency. Evidence:consumer-realms/evidence/acceptance-sdk205.json +SHA256a3240aaf1b3dd79efbcd8bdfd794481d0eb564b60748ddb2e23420884ab8718d; +activation-receipt.json records all exact digests/config/negative cases. + +Cleanup: acceptance clients auto-removed. Removed only two unmounted build +contexts13,893,772bytes; filesystem used198417211392->198403395584bytes. Recorded +image inventory; retained active Query0ed57643/Stream65918a65 and rollback +Query3dd54bfd/Stream7fe34806. No broad image/cache prune, no volumes or identities +deleted. Four-reader start times unchanged during cleanup. Canonical remains +/home/bobby/data_layer on fix/live-consumer-burst-recovery; release remainsv2.2.0, +not a newly published stable tag. No push/merge. BAR suffix source fba8863 is not +yet deployed to BAR writer.4000/s fullchain remains FAILED as recorded above; +2000/s exact captured transport/Redis/durableACK passed. diff --git a/rust/qdl-stream-gateway/examples/kn_trade_burst_gateway.rs b/rust/qdl-stream-gateway/examples/kn_trade_burst_gateway.rs new file mode 100644 index 00000000..1e0f8347 --- /dev/null +++ b/rust/qdl-stream-gateway/examples/kn_trade_burst_gateway.rs @@ -0,0 +1,545 @@ +//! Isolated capture-backed gRPC driver, NOT Kafka or live-freshness evidence. +//! Usage: kn_trade_burst_gateway CAPTURE TEMPLATES PRIVATE_DIR RATE REPEAT [PORT] +//! PRIVATE_DIR must exist; publication starts when every product is subscribed. +//! Packet credentials are ephemeral and written mode 0600, never to stdout. +use base64::{engine::general_purpose::STANDARD, Engine}; +use jsonwebtoken::{encode, Algorithm, DecodingKey, EncodingKey, Header}; +use prost::Message; +use qdl_contracts::cursor_v3::{ + CursorV3Claims, CursorV3Codec, CursorV3Expectation, DeliveryRequirement, +}; +use qdl_stream_gateway::{ + auth::{AccessError, JwtConfig, RequestQuota}, + authority::{Authority, AuthorityHandle}, + bundle::{Binding, Bundle, Manifest, ManifestRequirement, Quotas}, + generated::{ + marketdata_v2::{event_envelope, EventEnvelope}, + query_v2 as query, + }, + hub::{Hub, HubConfig, LogSource, RawRecord}, + readview::NotReadyReadView, + replay::{RangeCursor, RangeError, RangeSource, ReplayCoordinator}, + service::{Gateway, GatewayState, StreamLimits}, + subscription::ByteBudget, +}; +use query::market_data_stream_service_server::MarketDataStreamServiceServer; +use ring::{ + rand::{SecureRandom, SystemRandom}, + signature::{EcdsaKeyPair, KeyPair, ECDSA_P256_SHA256_FIXED_SIGNING}, +}; +use serde_json::{json, Value}; +use std::{ + collections::BTreeMap, + fs::{self, OpenOptions}, + io::{BufRead, BufReader, Write}, + os::unix::fs::OpenOptionsExt, + path::{Path, PathBuf}, + sync::{ + atomic::{AtomicU64, Ordering}, + Arc, + }, + time::{Duration, Instant, SystemTime, UNIX_EPOCH}, +}; + +const CONSUMER: &str = "test.capture.trade-burst"; +const SUBJECT: &str = "spiffe://test/capture-trade-burst"; +const STREAM: &str = "test.capture.canonical"; +fn now_ns() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos() as u64 +} +fn private_json(path: &Path, value: &Value) -> Result<(), Box> { + let temporary = path.with_extension("pending"); + let mut file = OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o600) + .open(&temporary)?; + file.write_all(serde_json::to_string(value)?.as_bytes())?; + file.sync_all()?; + fs::rename(temporary, path)?; + fs::File::open(path.parent().ok_or("output parent missing")?)?.sync_all()?; + Ok(()) +} + +struct Product { + uid: String, + envelope: EventEnvelope, + payloads: Vec>, + source_offsets: Vec<(i64, i64)>, + published: AtomicU64, +} +struct Capture { + products: Vec, +} +impl Capture { + fn record(&self, partition: i32, offset: i64) -> Option { + let product = self.products.get(usize::try_from(partition).ok()?)?; + if offset < 1 || offset as u64 > product.published.load(Ordering::Acquire) { + return None; + } + Some(RawRecord { + partition, + offset, + key: product.uid.as_bytes().to_vec(), + payload: product.payloads[(offset as usize - 1) % product.payloads.len()].clone(), + timestamp_ms: 0, + }) + } +} +struct Live { + capture: Arc, + positions: Vec, + next_partition: usize, +} +impl LogSource for Live { + fn poll(&mut self, timeout: Duration) -> Result, String> { + let deadline = Instant::now() + timeout; + loop { + for _ in 0..self.positions.len() { + let p = self.next_partition; + self.next_partition = (p + 1) % self.positions.len(); + if let Some(row) = self.capture.record(p as i32, self.positions[p]) { + self.positions[p] += 1; + return Ok(Some(row)); + } + } + if Instant::now() >= deadline { + return Ok(None); + } + std::thread::sleep(Duration::from_micros(250)); + } + } + fn positions(&self) -> Vec<(i32, i64)> { + self.positions + .iter() + .enumerate() + .map(|(p, &o)| (p as i32, o)) + .collect() + } +} +struct Ranges(Arc); +struct Reader { + capture: Arc, + partition: i32, + position: i64, +} +impl RangeSource for Ranges { + fn open(&self, partition: i32, from: i64) -> Result, RangeError> { + Ok(Box::new(Reader { + capture: self.0.clone(), + partition, + position: from.max(1), + })) + } +} +impl RangeCursor for Reader { + fn next(&mut self, _: Duration) -> Result, RangeError> { + let record = self.capture.record(self.partition, self.position); + if record.is_some() { + self.position += 1; + } + Ok(record) + } + fn position(&self) -> Option { + Some(self.position) + } +} +// Local fixture only: real JWT/manifest/cursor validation, no shared quota Redis. +struct FixtureQuota; +impl RequestQuota for FixtureQuota { + fn consume(&self, _: &Manifest) -> Result<(), AccessError> { + Ok(()) + } +} + +fn requirement(uid: &str, template: &Value) -> query::DataRequirement { + query::DataRequirement { + instrument_uid: uid.into(), + source_policy_id: template["requirement"]["source_policy_id"] + .as_str() + .unwrap() + .into(), + feed_type: query::FeedType::Trade as i32, + grade: query::ConsumerGrade::Execution as i32, + stale_policy_type: query::StalePolicy::Block as i32, + gap_policy_type: query::GapPolicy::Block as i32, + recovery_policy: query::RecoveryPolicy::SnapshotAndReplay as i32, + revision_policy: query::BarRevisionPolicy::Latest as i32, + require_full_coverage: true, + require_final_bars: true, + event_recency_policy: query::StalePolicy::Observe as i32, + max_freshness_ms: template["requirement"]["max_freshness_ms"] + .as_u64() + .unwrap(), + max_session_liveness_ms: template["requirement"]["max_session_liveness_ms"] + .as_u64() + .unwrap(), + // Actual TS OBSERVE delivery policy; only projection uses a test clock. + ..Default::default() + } +} + +#[tokio::main(flavor = "multi_thread", worker_threads = 2)] +async fn main() -> Result<(), Box> { + let args: Vec = std::env::args().collect(); + if args.len() < 6 || args.len() > 7 { + return Err("CAPTURE TEMPLATES PRIVATE_DIR RATE REPEAT [PORT]".into()); + } + let directory = PathBuf::from(&args[3]); + let rate: u64 = args[4].parse()?; + let repeat: u64 = args[5].parse()?; + if !(100..=4000).contains(&rate) || !(1..=30).contains(&repeat) { + return Err("bounded rate=100..4000 repeat=1..30 required".into()); + } + if directory.join("start").exists() || directory.join("ready.json").exists() { + return Err("fresh private directory required".into()); + } + let templates: Value = serde_json::from_slice(&fs::read(&args[2])?)?; + let mut products: Vec = Vec::new(); + let mut schedule = Vec::new(); + let mut bytes = 0usize; + for line in BufReader::new(fs::File::open(&args[1])?).lines() { + let row: Value = serde_json::from_str(&line?)?; + let payload = STANDARD.decode(row["payload"].as_str().ok_or("payload missing")?)?; + bytes += payload.len(); + if bytes > 32 << 20 || schedule.len() >= 20000 { + return Err("capture bound exceeded".into()); + } + let envelope = EventEnvelope::decode(payload.as_slice())?; + if !matches!(envelope.payload, Some(event_envelope::Payload::Trade(_))) + || templates.get(&envelope.instrument_uid).is_none() + { + return Err("TRADE and matching template required".into()); + } + let p = match products + .iter() + .position(|p| p.uid == envelope.instrument_uid) + { + Some(p) => p, + None => { + products.push(Product { + uid: envelope.instrument_uid.clone(), + envelope, + payloads: Vec::new(), + source_offsets: Vec::new(), + published: AtomicU64::new(0), + }); + products.len() - 1 + } + }; + let coordinate = ( + row["partition"].as_i64().ok_or("partition missing")?, + row["offset"].as_i64().ok_or("offset missing")?, + ); + if products[p] + .source_offsets + .last() + .is_some_and(|old| old.0 != coordinate.0 || old.1 >= coordinate.1) + { + return Err("capture product offsets must increase in one partition".into()); + } + products[p].payloads.push(payload); + products[p].source_offsets.push(coordinate); + schedule.push(p); + } + if schedule.is_empty() || products.len() > 10 { + return Err("expected 1..10 nonempty products".into()); + } + if schedule.len() as u64 * repeat > rate * 120 { + return Err("offer window exceeds 120 seconds".into()); + } + let capture = Arc::new(Capture { products }); + let rng = SystemRandom::new(); + let pkcs8 = EcdsaKeyPair::generate_pkcs8(&ECDSA_P256_SHA256_FIXED_SIGNING, &rng) + .map_err(|_| "key generation")?; + let key = EcdsaKeyPair::from_pkcs8(&ECDSA_P256_SHA256_FIXED_SIGNING, pkcs8.as_ref(), &rng) + .map_err(|_| "key parse")?; + let mut secret = vec![0; 32]; + rng.fill(&mut secret).map_err(|_| "random")?; + let codec = CursorV3Codec::new(&BTreeMap::from([("burst".into(), secret)]), "burst")?; + let expectation = CursorV3Expectation { + environment: "paper".into(), + stream: STREAM.into(), + source_topic_id: "capture-test-not-kafka".into(), + partition_plan_epoch: 1, + source_policy_revision: 1, + catalog_revision: 1, + route_generation: "capture-test-only".into(), + schema_major: 2, + }; + let mut bindings = Vec::new(); + let mut requirements = Vec::new(); + let mut routes = Vec::new(); + for (p, product) in capture.products.iter().enumerate() { + let req = requirement(&product.uid, &templates[&product.uid]); + let delivery = DeliveryRequirement::from_proto(&req)?; + let lpk = format!( + "lpk1|paper|{}|{}|{}|TRADE|-", + product.envelope.venue, product.envelope.market, product.uid + ); + bindings.push(Binding { + binding_id: format!("capture-{p}"), + instrument_uid: product.uid.clone(), + venue: product.envelope.venue.clone(), + market: product.envelope.market.clone(), + feed: "TRADE".into(), + interval: None, + source_policy_id: req.source_policy_id.clone(), + physical_key: product.uid.clone(), + product_key: lpk.clone(), + stale_after_ms: 3000, + }); + requirements.push(ManifestRequirement { + instrument_uid: product.uid.clone(), + feed: "TRADE".into(), + interval: None, + consumer_grade: "EXECUTION".into(), + source_policy_id: req.source_policy_id.clone(), + event_recency_policy: Some("OBSERVE".into()), + max_session_liveness_ms: Some(req.max_session_liveness_ms), + }); + let cursor = codec + .encode(&CursorV3Claims { + key_id: "burst".into(), + environment: "paper".into(), + consumer_id: CONSUMER.into(), + requirement_digest: delivery.digest(), + schema_major: 2, + stream: STREAM.into(), + product_key: lpk, + snapshot_id: "capture-start".into(), + source_topic_id: expectation.source_topic_id.clone(), + source_partition: p as u64, + source_offset: 0, + partition_plan_epoch: 1, + source_policy_revision: 1, + catalog_revision: 1, + route_generation: expectation.route_generation.clone(), + issued_at_ns: now_ns(), + expires_at_ns: now_ns() + 900_000_000_000, + }) + .map_err(|e| format!("cursor: {e:?}"))?; + routes.push(json!({"instrument_uid": product.uid, "partition": p, "cursor": cursor, + "requirement_proto_b64": STANDARD.encode(req.encode_to_vec()), "expected_events": product.payloads.len() as u64 * repeat, + "source_coordinates": product.source_offsets})); + } + let bundle = Bundle { + sha256: "capture-test-only".into(), + environment: "paper".into(), + catalog_revision: 1, + source_policy_revision: 1, + canonical_stream: STREAM.into(), + bindings, + manifests: vec![Manifest { + consumer_id: CONSUMER.into(), + subject: SUBJECT.into(), + environment: "paper".into(), + manifest_revision: 1, + allowed_purposes: vec!["INTERNAL_EXECUTION".into()], + allowed_permissions: vec!["stream:read".into(), "history:read".into()], + requirements, + quotas: Quotas { + requests_per_minute: 10000, + max_batch_items: 100, + max_warmup_rows: 5000, + max_streams: 32, + max_buffer_events: 1000, + }, + }], + }; + let authority = AuthorityHandle::new(Authority { + bundle, + expectation, + jwt: JwtConfig { + environment: "paper".into(), + issuer: "https://capture.test".into(), + audience: "capture-test".into(), + keys: BTreeMap::from([( + "burst".into(), + ( + DecodingKey::from_ec_der(key.public_key().as_ref()), + vec![Algorithm::ES256], + ), + )]), + algorithms: vec![Algorithm::ES256], + max_lifetime_seconds: 900, + environments_by_key_id: BTreeMap::new(), + subjects_by_key_id: BTreeMap::from([("burst".into(), SUBJECT.into())]), + }, + }); + let starts: Vec<_> = (0..capture.products.len()).map(|p| (p as i32, 1)).collect(); + let hub = Arc::new(Hub::new( + &starts, + HubConfig { + ring_max_bytes: (48 << 20) / starts.len(), + ring_max_age: Duration::from_secs(120), + }, + )); + let reader = hub.run(Box::new(Live { + capture: capture.clone(), + positions: vec![1; starts.len()], + next_partition: 0, + })); + let limits = StreamLimits::default(); + let state = Arc::new(GatewayState::new( + authority, + Arc::new(FixtureQuota), + codec, + hub.clone(), + ReplayCoordinator::new( + Arc::new(Ranges(capture.clone())), + 2, + limits.replay.clone(), + ByteBudget::new(32 << 20), + ), + Arc::new(NotReadyReadView), + ByteBudget::new(32 << 20), + limits, + )); + let port: u16 = args.get(6).map(String::as_str).unwrap_or("18219").parse()?; + let address: std::net::SocketAddr = format!("127.0.0.1:{port}").parse()?; + let tls_root = directory.join("tls"); + let tls_enabled = tls_root.exists(); + let encrypted = if tls_enabled { + let config = qdl_stream_gateway::tls::server_config( + &fs::read_to_string(tls_root.join("server.pem"))?, + &fs::read_to_string(tls_root.join("server.key"))?, + &fs::read_to_string(tls_root.join("ca.pem"))?, + )?; + Some(qdl_stream_gateway::tls::incoming(address, config).await?) + } else { + None + }; + let plaintext = if tls_enabled { + None + } else { + let listener = tokio::net::TcpListener::bind(address).await?; + let (tx, rx) = tokio::sync::mpsc::channel(8); + tokio::spawn(async move { + loop { + let result = listener.accept().await.map(|(s, _)| s); + if tx.send(result).await.is_err() { + break; + } + } + }); + Some(tokio_stream::wrappers::ReceiverStream::new(rx)) + }; + let tls_packet = if tls_enabled { + json!({"ca_file":"tls/ca.pem", "client_cert_file":"tls/client.pem", "client_key_file":"tls/client.key"}) + } else { + json!(false) + }; + let mut header = Header::new(Algorithm::ES256); + header.kid = Some("burst".into()); + let jwt = encode( + &header, + &json!({"sub": SUBJECT, "iss":"https://capture.test", "aud":"capture-test", + "iat": now_ns()/1_000_000_000, "exp": now_ns()/1_000_000_000+900, "jti": format!("capture-{}", now_ns()), + "environment":"paper", "roles":["stream_consumer","historical_reader","market_data_reader"], "consumer_manifest_revision":1}), + &EncodingKey::from_ec_der(pkcs8.as_ref()), + )?; + let cursors: BTreeMap<_, _> = routes + .iter() + .map(|r| (r["instrument_uid"].as_str().unwrap(), &r["cursor"])) + .collect(); + let totals: BTreeMap<_, _> = routes + .iter() + .map(|r| (r["instrument_uid"].as_str().unwrap(), &r["expected_events"])) + .collect(); + private_json( + &directory.join("ready.json"), + &json!({"target":format!("127.0.0.1:{port}"), "tls":tls_packet, + "test_only":true,"consumer_id":CONSUMER,"purpose":"INTERNAL_EXECUTION","token":jwt,"routes":routes,"cursors":cursors,"total_per_uid":totals, + "offered_rate":rate,"repeat":repeat,"capture_clock":"max(received_at_ns,source_event_time_ns)+1000000", + "scope":"test-only capture-backed real gRPC; no Kafka durability or live eligibility"}), + )?; + let stopping = state.clone(); + let publishing = state.clone(); + let output = directory.clone(); + let producer = tokio::spawn(async move { + let wait = Instant::now(); + while publishing.hub.subscriber_count() < capture.products.len() { + if wait.elapsed() > Duration::from_secs(90) { + return Err("start timeout".to_owned()); + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + let began = Instant::now(); + let total = schedule.len() as u64 * repeat; + let mut max_late = 0f64; + let mut max_lag = 0u64; + let mut queue_peak = 0usize; + for index in 0..total { + if index % 16 == 0 { + let due = Duration::from_secs_f64(index as f64 / rate as f64); + tokio::time::sleep(due.saturating_sub(began.elapsed())).await; + max_late = max_late.max(began.elapsed().saturating_sub(due).as_secs_f64() * 1000.0); + } + capture.products[schedule[index as usize % schedule.len()]] + .published + .fetch_add(1, Ordering::Release); + if index % 16 == 0 { + let lag: u64 = capture + .products + .iter() + .enumerate() + .map(|(p, product)| { + product.published.load(Ordering::Acquire).saturating_sub( + publishing + .hub + .next_offset(p as i32) + .unwrap_or(1) + .saturating_sub(1) as u64, + ) + }) + .sum(); + max_lag = max_lag.max(lag); + queue_peak = queue_peak.max(publishing.budget.used()); + } + } + private_json(&output.join("producer.json"), &json!({"offered":total,"offered_rate":rate, + "offer_seconds":began.elapsed().as_secs_f64(),"producer_schedule_late_max_ms":max_late, + "hub_lag_peak_sampled_records":max_lag,"queue_peak_sampled_bytes":queue_peak, + "pass":false,"note":"producer receipt only; TS apply/durable ACK and drain decide acceptance"})).map_err(|e| e.to_string())?; + let drain = Instant::now(); + while !output.join("stop").exists() && drain.elapsed() < Duration::from_secs(60) { + tokio::time::sleep(Duration::from_millis(50)).await; + } + let load = |v: &AtomicU64| v.load(Ordering::Relaxed); + private_json(&output.join("gateway.json"), &json!({"delivered":load(&publishing.metrics.delivered), + "replayed":load(&publishing.metrics.replayed),"overflow":load(&publishing.metrics.overflow), + "reader_errors":load(&publishing.hub.metrics.reader_errors),"decode_errors":load(&publishing.hub.metrics.decode_failures), + "queue_end_bytes":publishing.budget.used(),"drain_wait_seconds":drain.elapsed().as_secs_f64(), + "client_stop_received":output.join("stop").exists()})).map_err(|e| e.to_string())?; + Ok::<(), String>(()) + }); + let server = tonic::transport::Server::builder() + .add_service(MarketDataStreamServiceServer::new(Gateway { state })); + let shutdown = async move { + match producer.await { + Ok(Ok(())) => {} + other => eprintln!("capture producer failed: {other:?}"), + } + stopping.shut_down(); + }; + let result = match encrypted { + Some(incoming) => { + server + .serve_with_incoming_shutdown(incoming, shutdown) + .await + } + None => { + server + .serve_with_incoming_shutdown(plaintext.unwrap(), shutdown) + .await + } + }; + hub.stop(); + reader.join().map_err(|_| "reader panicked")?; + result?; + Ok(()) +} diff --git a/scripts/accept_consumer_realms.py b/scripts/accept_consumer_realms.py new file mode 100644 index 00000000..2063e348 --- /dev/null +++ b/scripts/accept_consumer_realms.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python3 +"""Strict no-order Query/Stream acceptance for separately pinned consumer realms.""" +import argparse +import asyncio +import json +import time +from pathlib import Path + +import yaml +from qdl_sdk import DataRequirement, Feed, Grade, StalePolicy, StreamEvent +from qdl_sdk.client import AsyncDataLayerClient +from qdl_sdk.credentials import RotatingJwtCredentialProvider +from qdl_sdk.errors import DataLayerError +from qdl_sdk.models import GapPolicy, RecoveryPolicy, BarRevisionPolicy +from qdl_sdk.tls import WorkloadTlsConfig +from qdl_sdk.transport import RestQueryTransport, GrpcStreamTransport + + +def requirement(row): + return DataRequirement(instrument_uid=row["instrument_uid"], feed=Feed(row["feed"]), + consumer_grade=Grade(row["consumer_grade"]), source_policy_id=row["source_policy_id"], + interval=row.get("interval"), warmup_limit=0, max_freshness_ms=row.get("max_freshness_ms"), + event_recency_policy=StalePolicy(row["event_recency_policy"]) if row.get("event_recency_policy") else None, + max_session_liveness_ms=row.get("max_session_liveness_ms"), + require_full_coverage=row.get("require_full_coverage", True), + require_final_bars=row.get("require_final_bars", True), + stale_policy=StalePolicy(row.get("stale_policy", "BLOCK")), + gap_policy=GapPolicy(row.get("gap_policy", "BLOCK")), + recovery=RecoveryPolicy(row.get("recovery", "SNAPSHOT_AND_REPLAY")), + bar_revision_policy=BarRevisionPolicy(row.get("bar_revision_policy", "LATEST"))) + + +async def run(args): + packet = Path(args.packet) + policy = json.loads((packet / "reader-public/jwt-config.json").read_text()) + reads, negatives, streams = [], [], [] + for path in sorted((packet / "manifests-crypto").glob("*.yaml")): + payload = yaml.safe_load(path.read_text()) + meta, declared = payload["metadata"], payload["spec"]["requirements"] + cid, realm = meta["id"], meta["environment"] + identity = packet / "identities" / cid + tls = WorkloadTlsConfig(identity / "ca.crt", identity / "client.crt", identity / "client.key") + def credential(**overrides): + values = dict(private_key_file=identity / "private.key", key_id=cid + "-rs256-v1", + algorithm="RS256", issuer=policy["issuer"], audience=policy["audience"], + subject=meta["subject"], environment=realm, + roles=("market_data_reader", "historical_reader", "stream_consumer"), + consumer_manifest_revision=meta["revision"], lifetime_seconds=300, refresh_before_seconds=30) + values.update(overrides) + return RotatingJwtCredentialProvider(**values) + selected = list(declared) if cid.startswith("trading-system.") else [] + if not selected: + for feed in ("TRADE", "QUOTE", "BAR", "MARK_INDEX_PRICE", "BOOK_SNAPSHOT", "BOOK_DELTA"): + row = next((r for r in declared if r["feed"] == feed and (feed != "BAR" or r.get("interval") == "1m")), None) + if row: + selected.append(row) + for replica in (1, 2): + creds = credential() + url = f"https://query_v2_{replica}:8200" + query = RestQueryTransport(url, credential_provider=creds, tls=tls) + transport = GrpcStreamTransport(f"qdl-v2-stream-{'a' if replica == 1 else 'b'}:8210", credential_provider=creds, tls=tls) + client = AsyncDataLayerClient(query_transport=query, stream_transport=transport, + consumer_id=cid, max_warmup_attempts=1) + try: + for raw in selected: + req = requirement(raw) + start = time.perf_counter() + row = dict(consumer_id=cid, realm=realm, replica=replica, + instrument_uid=req.instrument_uid, feed=req.feed.value, interval=req.interval) + try: + response = await client.snapshot(req) + quality = response.data.quality + row.update(status="TYPED_RESPONSE", state=quality.state, + execution_eligible=quality.execution_eligible, + event_recency_state=quality.event_recency_state, + provider_session_state=quality.provider_session_state, + gap_open=quality.gap_open, complete=quality.complete) + except Exception as error: + row.update(status="REFUSED", code=getattr(error, "code", type(error).__name__), detail=str(error)[:180]) + row["call_to_result_ms"] = (time.perf_counter() - start) * 1000 + reads.append(row) + await asyncio.sleep(.1) + probe = requirement(next(r for r in selected if r["feed"] == "TRADE")) + for name, kwargs, consumer in ( + ("wrong-realm", {"environment": "paper"}, cid), + ("wrong-revision", {"consumer_manifest_revision": meta["revision"] + 1}, cid), + ("wrong-consumer", {}, cid.replace("." + realm + ".", ".paper.")), + ): + bad = RestQueryTransport(url, credential_provider=credential(**kwargs), tls=tls) + try: + await bad.snapshot(probe, consumer_id=consumer) + negatives.append(dict(consumer_id=cid, replica=replica, case=name, refused=False)) + except DataLayerError as error: + negatives.append(dict(consumer_id=cid, replica=replica, case=name, + refused=error.code in {"UNAUTHENTICATED", "CONSUMER_MISMATCH", "PERMISSION_DENIED"}, code=error.code)) + finally: + await bad.close() + try: + async with asyncio.timeout(20): + async with client.warmup_then_stream(probe) as session: + async for event in session: + if isinstance(event, StreamEvent): + await session.acknowledge_async(event) + streams.append(dict(consumer_id=cid, replica=replica, status="STREAM_ACK_PASS")) + break + except Exception as error: + streams.append(dict(consumer_id=cid, replica=replica, status="FAILED", + code=getattr(error, "code", type(error).__name__), detail=str(error)[:180])) + finally: + await client.close() + # No DATA_STALE exemption and no retry seeking a green sample. + failures = [r for r in reads if r["status"] != "TYPED_RESPONSE"] + usable = [r for r in reads if r.get("execution_eligible") is True] + report = dict(schema="qdl.consumer-realm-acceptance.v1", order_actions=0, + snapshots=reads, negatives=negatives, streams=streams, + typed_response_gate=bool(reads) and not failures, + execution_usable_count=len(usable), total_reads=len(reads), + negative_gate=bool(negatives) and all(r["refused"] for r in negatives), + stream_gate=len(streams) == 12 and all(r["status"] == "STREAM_ACK_PASS" for r in streams)) + report["all_reads_execution_eligible"] = bool(reads) and len(usable) == len(reads) + Path(args.output).write_text(json.dumps(report, indent=2)) + print(json.dumps({k:v for k,v in report.items() if k not in {"snapshots", "negatives", "streams"}})) + return report["typed_response_gate"] and report["negative_gate"] and report["stream_gate"] + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--packet", required=True) + parser.add_argument("--output", required=True) + raise SystemExit(0 if asyncio.run(run(parser.parse_args())) else 1) diff --git a/scripts/kn_trade_burst_build.sh b/scripts/kn_trade_burst_build.sh new file mode 100644 index 00000000..b5c3c03d --- /dev/null +++ b/scripts/kn_trade_burst_build.sh @@ -0,0 +1,7 @@ +#!/bin/sh +# Run inside the existing Rust builder with --cpus 2 --memory 2g --memory-swap 2g. +set -eu +export CARGO_BUILD_JOBS=2 +rustfmt --edition 2021 rust/qdl-stream-gateway/examples/kn_trade_burst_gateway.rs +cargo test --locked -j 2 -p qdl-stream-gateway --lib --test native_stream -- --test-threads=2 +cargo build --locked -j 2 -p qdl-stream-gateway --example kn_trade_burst_gateway diff --git a/scripts/kn_trade_burst_tls.py b/scripts/kn_trade_burst_tls.py new file mode 100644 index 00000000..1df72a88 --- /dev/null +++ b/scripts/kn_trade_burst_tls.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 +"""Create a fresh two-hour test CA and mTLS identities in PRIVATE_DIR/tls.""" +import argparse +from datetime import datetime, timedelta, timezone +from ipaddress import ip_address +import os +from pathlib import Path + +from cryptography import x509 +from cryptography.hazmat.primitives import hashes, serialization +from cryptography.hazmat.primitives.asymmetric import ec +from cryptography.x509.oid import ExtendedKeyUsageOID, NameOID + + +def create(directory): + root = Path(directory) / "tls" + root.mkdir(mode=0o700) + now = datetime.now(timezone.utc) + ca_key = ec.generate_private_key(ec.SECP256R1()) + ca_name = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, "capture-test-only-ca")]) + + def builder(subject, key): + return (x509.CertificateBuilder().subject_name(subject).issuer_name(ca_name) + .public_key(key.public_key()).serial_number(x509.random_serial_number()) + .not_valid_before(now - timedelta(minutes=1)) + .not_valid_after(now + timedelta(hours=2))) + + def write(name, data): + descriptor = os.open(root / name, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(descriptor, "wb") as handle: + handle.write(data) + handle.flush() + os.fsync(handle.fileno()) + + ca = (builder(ca_name, ca_key).add_extension(x509.BasicConstraints(ca=True, path_length=0), True) + .add_extension(x509.KeyUsage(digital_signature=True, content_commitment=False, + key_encipherment=False, data_encipherment=False, key_agreement=False, + key_cert_sign=True, crl_sign=True, encipher_only=False, decipher_only=False), True) + .sign(ca_key, hashes.SHA256())) + write("ca.pem", ca.public_bytes(serialization.Encoding.PEM)) + for name, usage in (("server", ExtendedKeyUsageOID.SERVER_AUTH), + ("client", ExtendedKeyUsageOID.CLIENT_AUTH)): + key = ec.generate_private_key(ec.SECP256R1()) + subject = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, "capture-test-" + name)]) + certificate = (builder(subject, key) + .add_extension(x509.BasicConstraints(ca=False, path_length=None), True) + .add_extension(x509.ExtendedKeyUsage([usage]), False)) + if name == "server": + certificate = certificate.add_extension(x509.SubjectAlternativeName( + [x509.IPAddress(ip_address("127.0.0.1")), x509.DNSName("localhost")]), False) + certificate = certificate.sign(ca_key, hashes.SHA256()) + write(name + ".pem", certificate.public_bytes(serialization.Encoding.PEM)) + write(name + ".key", key.private_bytes(serialization.Encoding.PEM, + serialization.PrivateFormat.PKCS8, serialization.NoEncryption())) + # CA private key is deliberately never persisted. + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("directory") + create(parser.parse_args().directory) diff --git a/scripts/kn_trade_burst_wire_smoke.py b/scripts/kn_trade_burst_wire_smoke.py new file mode 100644 index 00000000..3b42e64f --- /dev/null +++ b/scripts/kn_trade_burst_wire_smoke.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 +"""Bounded wire-only driver regression; not SDK/TS/Redis/ACK acceptance.""" +import argparse +import asyncio +import base64 +import json +from pathlib import Path +import tempfile + +import grpc +from qdl.marketdata.v2.market_data_pb2 import EventEnvelope +from qdl.query.v2 import query_pb2 as pb + + +async def run(args): + expected = {} + for line in Path(args.capture).read_text().splitlines(): + payload = base64.b64decode(json.loads(line)["payload"], validate=True) + uid = EventEnvelope.FromString(payload).instrument_uid + expected.setdefault(uid, []).append(payload) + with tempfile.TemporaryDirectory(prefix="kn-trade-wire-") as directory: + root = Path(directory) + process = await asyncio.create_subprocess_exec( + args.driver, args.capture, args.templates, directory, "4000", "1", str(args.port), + stdout=asyncio.subprocess.DEVNULL, stderr=asyncio.subprocess.PIPE) + tasks = [] + try: + async with asyncio.timeout(30): + while not (root / "ready.json").exists(): + if process.returncode is not None: + raise RuntimeError("driver exited before readiness") + await asyncio.sleep(0.02) + packet = json.loads((root / "ready.json").read_text()) + assert set(packet["total_per_uid"]) == set(expected) + ready = {uid: asyncio.Event() for uid in expected} + metadata = (("authorization", "Bearer " + packet["token"]), + ("x-qdl-consumer-id", packet["consumer_id"]), + ("x-qdl-purpose", packet["purpose"])) + async with grpc.aio.insecure_channel(packet["target"]) as channel: + rpc = channel.unary_stream( + "/qdl.query.v2.MarketDataStreamService/Subscribe", + request_serializer=pb.SubscribeRequest.SerializeToString, + response_deserializer=pb.SubscribeResponse.FromString) + + async def consume(route): + uid = route["instrument_uid"] + req = pb.DataRequirement.FromString(base64.b64decode(route["requirement_proto_b64"])) + call = rpc(pb.SubscribeRequest(consumer_id=packet["consumer_id"], requirement=req, + cursor_token=route["cursor"], max_buffer_events=1000), metadata=metadata) + count = 0 + try: + async for response in call: + record = response.record + if record.HasField("control"): + if pb.StreamControlState.Name(record.control.state).endswith("LIVE"): + ready[uid].set() + continue + if not record.HasField("event"): + raise AssertionError("unexpected record") + assert record.logical_offset == count + 1 + assert record.event.SerializeToString(deterministic=True) == expected[uid][count] + count += 1 + if count == len(expected[uid]): + return count + raise AssertionError("early stream end") + finally: + call.cancel() + + tasks = [asyncio.create_task(consume(route)) for route in packet["routes"]] + await asyncio.gather(*(event.wait() for event in ready.values())) + (root / "start").touch() + counts = await asyncio.gather(*tasks) + (root / "stop").touch() + assert await process.wait() == 0 + metrics = json.loads((root / "gateway.json").read_text()) + assert metrics["overflow"] == metrics["reader_errors"] == metrics["decode_errors"] == 0 + print(json.dumps({"pass": True, "scope": "wire-only; no SDK/apply/ACK claim", + "products": len(counts), "events": sum(counts), + "producer": json.loads((root / "producer.json").read_text()), + "gateway": metrics})) + finally: + for task in tasks: + task.cancel() + await asyncio.gather(*tasks, return_exceptions=True) + if process.returncode is None: + process.terminate() + await process.wait() + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--driver", required=True) + parser.add_argument("--capture", required=True) + parser.add_argument("--templates", required=True) + parser.add_argument("--port", type=int, default=18229) + asyncio.run(run(parser.parse_args())) diff --git a/upgrade/DATA_LAYER_V2_KAFKA_NATIVE_ARCHITECTURE_REVIEW.md b/upgrade/DATA_LAYER_V2_KAFKA_NATIVE_ARCHITECTURE_REVIEW.md index 9e23e2d9..0e4491fe 100644 --- a/upgrade/DATA_LAYER_V2_KAFKA_NATIVE_ARCHITECTURE_REVIEW.md +++ b/upgrade/DATA_LAYER_V2_KAFKA_NATIVE_ARCHITECTURE_REVIEW.md @@ -3018,3 +3018,28 @@ scopes. TS PAPER_ONLY becomes ALLOWED for its explicitly approved sandbox/live read identity. Alpha FORBIDDEN direct execution dependency stays FORBIDDEN: alpha uses data to construct intent, while TS Risk remains the admission authority. Read activation never enables orders, changes broker mode, or starts an alpha. + +#### Three-realm operation and consumer handoff + +Adding an approved reader is configuration materialization, not a new service or +per-symbol image. Compile the versioned manifest with compile_consumer_realms.py, +provision a separate TLS identity and JWT key, and include the public key/subject/ +realm and manifest in the reader packet. Current Query startup requires a bounded +reader reload/recreate for configuration changes; do not promise zero reload. +No code/image rebuild is needed merely for another binding within supported +contracts. New feed implementations still require source tests and release. + +Consumer identity, sealed binding, manifest revision and credential must change +together. TS settings use DATA_LAYER_V2_JWT_KEY_ID, JWT_SUBJECT, JWT_ENVIRONMENT, +JWT_MANIFEST_REVISION and JWT_PRIVATE_KEY_FILE (all prefixed DATA_LAYER_V2_), plus +TLS_CA_FILE/TLS_CERT_FILE/TLS_KEY_FILE and CONSUMER_BINDING_FILE. Use the real +configuration field names in each consumer; do not rename a paper key to live. +Keep separate cursor/audit paths by consumer ID and preserve source-policy gates. +The runtime packet under consumer-realms contains private credentials outside Git; +mount only the selected consumer identity, never the whole identity directory. + +The realm labels workload permission only: sandbox/live still read the same +public mainnet market products. They do not select demo exchange books, change +account mode, establish broker execution parity, or permit order submission. +Current TS process remains on its existing paper read identity until a consumer +config handoff; the additional server mappings are active and tested no-order. From 6ac1a606804fbe7cace3ee7d5bedf252d32853f8 Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Sun, 27 Sep 2026 18:29:19 +0000 Subject: [PATCH 06/17] Preserve exact rejected quality and bound native5000 burst profiles --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 31 +++++++++ qdl_sdk/client.py | 44 ++++++++---- .../examples/kn_trade_burst_gateway.rs | 67 +++++++++++++++++-- scripts/accept_consumer_realms.py | 11 ++- tests/test_qdl_sdk_stream_projection.py | 47 +++++++++++++ 5 files changed, 179 insertions(+), 21 deletions(-) diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index 18d8d326..e1478d84 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -60440,3 +60440,34 @@ deleted. Four-reader start times unchanged during cleanup. Canonical remains not a newly published stable tag. No push/merge. BAR suffix source fba8863 is not yet deployed to BAR writer.4000/s fullchain remains FAILED as recorded above; 2000/s exact captured transport/Redis/durableACK passed. + +#### Approved exact-quality and production-adapter capacity closure + +Status: IN_PROGRESS. Owner excludes Binance 3d from this closure (no data deletion +or binding disable). Guide KN18.12-18.13 and post-release realms apply. Preserve +provider timestamps, raw payload provenance, quality gates, ACK-after-apply and +all order services. Capture the exact rejected view and match trade/component +source evidence, not a subsequent successful read. Use the actual TS production +adapter batching/recovery path in isolated 4000/5000 total-event/s tests; profile +before optimizing, no larger buffers to conceal backlog. Test steady/burst/drain, +disconnect/replay, slow-reader isolation, exact ordering/idempotency/cursor state, +and latency through consumer Redis. Quiet stale TRADE must remain non-executable. +Full requires explicit positive fresh-data, expected negative quality, lineage, +capacity and recovery evidence; no certificate for unmeasured fan-out or Kafka +ingest capacity. No new phase/topology, no alpha/order activation. Record exact +per-role runtime packet/rollback before any required deploy. Cleanup only owned +test resources; preserve active/rollback artifacts and owner working-tree edits. + +Exact-quality slice checkpoint: SDK validation now attaches the rejected typed +view quality/source/contract/timestamps/watermark to existing ContinuityError +without changing its code or policy. Payload and signed cursor are excluded. +Acceptance tool keeps exact quality also for successful non-executable views. +Projection/quality unittest13PASS includes immutable error evidence and preserved +stale/disconnect/gap behavior. Runtime image lacks pytest; use existing unittest +runner, no package installation. Isolated driver now supports bounded5000/s, +repeat120 with scheduled offer<=120s and explicit low/high/low profile; original +provider bytes remain unchanged. New scheduling unit1PASS and release buildPASS. +Initial builder attempts failed toolchain PATH, then wrong Cargo cache layout; +correct existing registry mount is qdl-cargo-home:/usr/local/cargo/registry. +No production rollout or test capacity claim from these source/build results. +SDK stream/cursor unittest26PASS; combined affected unit count39PASS. diff --git a/qdl_sdk/client.py b/qdl_sdk/client.py index 28b1ea3f..43e33b24 100644 --- a/qdl_sdk/client.py +++ b/qdl_sdk/client.py @@ -128,6 +128,24 @@ def _allows_quiet_execution_continuity(requirement: DataRequirement, quality) -> ) +def _view_refusal(row, code: str, detail: str) -> ContinuityError: + """Retain the rejected typed view, excluding payload and signed cursor.""" + return ContinuityError(code, detail, diagnostics={ + "origin": "sdk_query_validation", + "instrument_uid": row.instrument_uid, + "instrument_id": row.instrument_id, + "feed": row.feed.value, + "interval": row.interval, + "observed_at_ns": row.observed_at_ns, + "received_at_ns": row.received_at_ns, + "watermark_offset": row.watermark_offset, + "revision": row.revision, + "quality": row.quality.model_dump(mode="json"), + "source": row.source.model_dump(mode="json"), + "contract": row.contract.model_dump(mode="json"), + }) + + def _validate_query_payload( requirement: DataRequirement, payload: dict | WarmupResponse | SnapshotResponse, *, warmup: bool ) -> SnapshotResponse | WarmupResponse: @@ -165,18 +183,18 @@ def _validate_query_payload( for index, row in enumerate(rows): is_tail = index == len(rows) - 1 if row.instrument_uid != requirement.instrument_uid: - raise ContinuityError("CONFLICT", "query response instrument does not match requirement") + raise _view_refusal(row, "CONFLICT", "query response instrument does not match requirement") if row.feed.value != requirement.feed.value: - raise ContinuityError("CONFLICT", "query response feed does not match requirement") + raise _view_refusal(row, "CONFLICT", "query response feed does not match requirement") if requirement.interval is not None and row.interval != requirement.interval: - raise ContinuityError("CONFLICT", "query response interval does not match requirement") + raise _view_refusal(row, "CONFLICT", "query response interval does not match requirement") quality = row.quality if requirement.feed is Feed.BOOK_DELTA and ( not row.payload.sequence_verified or row.payload.book_generation < 1 ): - raise ContinuityError("OPEN_SEQUENCE_GAP", "book delta sequence is not verified") + raise _view_refusal(row, "OPEN_SEQUENCE_GAP", "book delta sequence is not verified") if quality.policy_id != requirement.source_policy_id: - raise ContinuityError( + raise _view_refusal(row, "CONFLICT", "query response source policy does not match requirement" ) state = quality.state.upper() @@ -188,12 +206,12 @@ def _validate_query_payload( and freshness_ms > requirement.max_freshness_ms and requirement.effective_event_recency_policy.value in {"BLOCK", "PAUSE"} ): - raise ContinuityError("DATA_STALE", "query response exceeds freshness policy") + raise _view_refusal(row, "DATA_STALE", "query response exceeds freshness policy") if ( is_tail and quality.provider_session_state in {"STALE", "DISCONNECTED", "UNKNOWN"} ): - raise ContinuityError( + raise _view_refusal(row, "DATA_STALE", "query response provider session is not live" ) if is_tail and requirement.max_session_liveness_ms is not None and ( @@ -202,33 +220,33 @@ def _validate_query_payload( or quality.provider_session_liveness_ms > requirement.max_session_liveness_ms ): - raise ContinuityError( + raise _view_refusal(row, "DATA_STALE", "query response provider session exceeds its policy" ) if quality.gap_open and requirement.gap_policy.value in {"BLOCK", "PAUSE"}: - raise ContinuityError("OPEN_SEQUENCE_GAP", "query response has an open gap") + raise _view_refusal(row, "OPEN_SEQUENCE_GAP", "query response has an open gap") if is_tail and state in { "STALE", "OFFLINE", "UNAVAILABLE" } and requirement.stale_policy.value in { "BLOCK", "PAUSE", }: - raise ContinuityError("DATA_STALE", f"query response quality state is {state}") + raise _view_refusal(row, "DATA_STALE", f"query response quality state is {state}") if ( is_tail and requirement.consumer_grade is Grade.EXECUTION and not quality.execution_eligible and not _allows_quiet_execution_continuity(requirement, quality) ): - raise ContinuityError( + raise _view_refusal(row, "SOURCE_NON_AUTHORITATIVE", "execution-grade response is not execution eligible", ) if requirement.require_full_coverage and not quality.complete: - raise ContinuityError("PARTIAL_RESULT", "query response quality is incomplete") + raise _view_refusal(row, "PARTIAL_RESULT", "query response quality is incomplete") if requirement.feed is Feed.BAR and requirement.require_final_bars: lifecycle = str(getattr(row.payload, "lifecycle", "")).upper() if lifecycle not in {"FINAL", "REVISED"}: - raise ContinuityError("DATA_NOT_READY", "bar response is not final") + raise _view_refusal(row, "DATA_NOT_READY", "bar response is not final") if requirement.feed is Feed.BAR: opens = [int(row.payload.open_time_ns) for row in rows] if opens != sorted(set(opens)): diff --git a/rust/qdl-stream-gateway/examples/kn_trade_burst_gateway.rs b/rust/qdl-stream-gateway/examples/kn_trade_burst_gateway.rs index 1e0f8347..958c2e2c 100644 --- a/rust/qdl-stream-gateway/examples/kn_trade_burst_gateway.rs +++ b/rust/qdl-stream-gateway/examples/kn_trade_burst_gateway.rs @@ -44,6 +44,20 @@ use std::{ const CONSUMER: &str = "test.capture.trade-burst"; const SUBJECT: &str = "spiffe://test/capture-trade-burst"; const STREAM: &str = "test.capture.canonical"; +fn scheduled_seconds(index: u64, total: u64, rate: u64, low_rate: u64) -> f64 { + let quarter = total / 4; + let middle_end = total - quarter; + if index <= quarter { + index as f64 / low_rate as f64 + } else if index <= middle_end { + quarter as f64 / low_rate as f64 + (index - quarter) as f64 / rate as f64 + } else { + quarter as f64 / low_rate as f64 + + (middle_end - quarter) as f64 / rate as f64 + + (index - middle_end) as f64 / low_rate as f64 + } +} + fn now_ns() -> u64 { SystemTime::now() .duration_since(UNIX_EPOCH) @@ -191,8 +205,16 @@ async fn main() -> Result<(), Box> { let directory = PathBuf::from(&args[3]); let rate: u64 = args[4].parse()?; let repeat: u64 = args[5].parse()?; - if !(100..=4000).contains(&rate) || !(1..=30).contains(&repeat) { - return Err("bounded rate=100..4000 repeat=1..30 required".into()); + if !(100..=5000).contains(&rate) || !(1..=120).contains(&repeat) { + return Err("bounded rate=100..5000 repeat=1..120 required".into()); + } + let low_rate: u64 = std::env::var("QDL_TEST_BURST_LOW_RATE") + .ok() + .map(|v| v.parse()) + .transpose()? + .unwrap_or(rate); + if !(100..=rate).contains(&low_rate) { + return Err("bounded burst low rate=100..offered rate required".into()); } if directory.join("start").exists() || directory.join("ready.json").exists() { return Err("fresh private directory required".into()); @@ -248,7 +270,13 @@ async fn main() -> Result<(), Box> { if schedule.is_empty() || products.len() > 10 { return Err("expected 1..10 nonempty products".into()); } - if schedule.len() as u64 * repeat > rate * 120 { + if scheduled_seconds( + schedule.len() as u64 * repeat, + schedule.len() as u64 * repeat, + rate, + low_rate, + ) > 120.0 + { return Err("offer window exceeds 120 seconds".into()); } let capture = Arc::new(Capture { products }); @@ -454,7 +482,7 @@ async fn main() -> Result<(), Box> { &directory.join("ready.json"), &json!({"target":format!("127.0.0.1:{port}"), "tls":tls_packet, "test_only":true,"consumer_id":CONSUMER,"purpose":"INTERNAL_EXECUTION","token":jwt,"routes":routes,"cursors":cursors,"total_per_uid":totals, - "offered_rate":rate,"repeat":repeat,"capture_clock":"max(received_at_ns,source_event_time_ns)+1000000", + "offered_rate":rate,"burst_low_rate":low_rate,"repeat":repeat,"capture_clock":"max(received_at_ns,source_event_time_ns)+1000000", "scope":"test-only capture-backed real gRPC; no Kafka durability or live eligibility"}), )?; let stopping = state.clone(); @@ -470,12 +498,21 @@ async fn main() -> Result<(), Box> { } let began = Instant::now(); let total = schedule.len() as u64 * repeat; + let started_at_ns = now_ns(); + private_json( + &output.join("offer-start.json"), + &json!({ + "started_at_ns":started_at_ns,"total":total,"rate":rate,"burst_low_rate":low_rate, + "clock":"test scheduled injection wall clock; not original provider time" + }), + ) + .map_err(|e| e.to_string())?; let mut max_late = 0f64; let mut max_lag = 0u64; let mut queue_peak = 0usize; for index in 0..total { if index % 16 == 0 { - let due = Duration::from_secs_f64(index as f64 / rate as f64); + let due = Duration::from_secs_f64(scheduled_seconds(index, total, rate, low_rate)); tokio::time::sleep(due.saturating_sub(began.elapsed())).await; max_late = max_late.max(began.elapsed().saturating_sub(due).as_secs_f64() * 1000.0); } @@ -501,7 +538,7 @@ async fn main() -> Result<(), Box> { queue_peak = queue_peak.max(publishing.budget.used()); } } - private_json(&output.join("producer.json"), &json!({"offered":total,"offered_rate":rate, + private_json(&output.join("producer.json"), &json!({"offered":total,"offered_rate":rate,"burst_low_rate":low_rate,"started_at_ns":started_at_ns, "offer_seconds":began.elapsed().as_secs_f64(),"producer_schedule_late_max_ms":max_late, "hub_lag_peak_sampled_records":max_lag,"queue_peak_sampled_bytes":queue_peak, "pass":false,"note":"producer receipt only; TS apply/durable ACK and drain decide acceptance"})).map_err(|e| e.to_string())?; @@ -543,3 +580,21 @@ async fn main() -> Result<(), Box> { result?; Ok(()) } + +#[cfg(test)] +mod scheduling_tests { + use super::scheduled_seconds; + #[test] + fn constant_and_burst_are_bounded_monotonic_schedules() { + assert_eq!(scheduled_seconds(10000, 10000, 5000, 5000), 2.0); + assert_eq!(scheduled_seconds(2500, 10000, 5000, 1000), 2.5); + assert_eq!(scheduled_seconds(7500, 10000, 5000, 1000), 3.5); + assert_eq!(scheduled_seconds(10000, 10000, 5000, 1000), 6.0); + for index in 1..=10003 { + assert!( + scheduled_seconds(index, 10003, 5000, 1000) + > scheduled_seconds(index - 1, 10003, 5000, 1000) + ); + } + } +} diff --git a/scripts/accept_consumer_realms.py b/scripts/accept_consumer_realms.py index 2063e348..f0c5d85b 100644 --- a/scripts/accept_consumer_realms.py +++ b/scripts/accept_consumer_realms.py @@ -74,9 +74,16 @@ def credential(**overrides): execution_eligible=quality.execution_eligible, event_recency_state=quality.event_recency_state, provider_session_state=quality.provider_session_state, - gap_open=quality.gap_open, complete=quality.complete) + gap_open=quality.gap_open, complete=quality.complete, + sampled_at_ns=time.time_ns(), + observed_at_ns=response.data.observed_at_ns, + received_at_ns=response.data.received_at_ns, + watermark_offset=response.data.watermark_offset, + quality=quality.model_dump(mode="json"), + source=response.data.source.model_dump(mode="json"), + contract=response.data.contract.model_dump(mode="json")) except Exception as error: - row.update(status="REFUSED", code=getattr(error, "code", type(error).__name__), detail=str(error)[:180]) + row.update(status="REFUSED", code=getattr(error, "code", type(error).__name__), detail=str(error)[:180], diagnostics=getattr(error, "diagnostics", None)) row["call_to_result_ms"] = (time.perf_counter() - start) * 1000 reads.append(row) await asyncio.sleep(.1) diff --git a/tests/test_qdl_sdk_stream_projection.py b/tests/test_qdl_sdk_stream_projection.py index e89f86e7..edfbfb74 100644 --- a/tests/test_qdl_sdk_stream_projection.py +++ b/tests/test_qdl_sdk_stream_projection.py @@ -187,6 +187,53 @@ def template(feed: Feed) -> MarketDataView: ) +class ExactQueryRefusalTests(unittest.TestCase): + def test_rejection_keeps_original_quality_not_later_view(self): + for feed in (Feed.TRADE, Feed.QUOTE, Feed.MARK_INDEX_PRICE, Feed.BOOK_DELTA): + with self.subTest(feed=feed): + view = template(feed) + view.quality.execution_eligible = False + view.quality.flags = ["component:index:stale", "generation:7"] + view.received_at_ns = NOW + 1000 + view.cursor = "signed-private-cursor-not-for-logs" + requirement = DataRequirement(instrument_uid=view.instrument_uid, + feed=feed, consumer_grade=Grade.EXECUTION, + source_policy_id="crypto_primary_v2", max_freshness_ms=2000) + with self.assertRaises(ContinuityError) as caught: + _validate_query_payload(requirement, + {"request_id": "exact", "data": view}, warmup=False) + evidence = caught.exception.diagnostics + self.assertEqual(caught.exception.code, "SOURCE_NON_AUTHORITATIVE") + self.assertEqual(evidence["quality"], view.quality.model_dump(mode="json")) + self.assertEqual(evidence["watermark_offset"], 10) + self.assertEqual(evidence["received_at_ns"], NOW + 1000) + self.assertNotIn("cursor", evidence) + self.assertNotIn("payload", evidence) + view.quality.flags.clear() + view.quality.execution_eligible = True + self.assertFalse(evidence["quality"]["execution_eligible"]) + self.assertEqual(len(evidence["quality"]["flags"]), 2) + + def test_stale_disconnected_and_gap_diagnostics_keep_error_codes(self): + cases = ( + ({"freshness_ms": 3001}, "DATA_STALE"), + ({"provider_session_state": "DISCONNECTED"}, "DATA_STALE"), + ({"gap_open": True}, "OPEN_SEQUENCE_GAP"), + ) + for changes, code in cases: + view = template(Feed.TRADE) + view.quality = view.quality.model_copy(update=changes) + requirement = DataRequirement(instrument_uid=view.instrument_uid, + feed=Feed.TRADE, consumer_grade=Grade.EXECUTION, + source_policy_id="crypto_primary_v2", max_freshness_ms=2000) + with self.assertRaises(ContinuityError) as caught: + _validate_query_payload(requirement, + {"request_id": "negative", "data": view}, warmup=False) + self.assertEqual(caught.exception.code, code) + self.assertEqual(caught.exception.diagnostics["quality"], + view.quality.model_dump(mode="json")) + + def envelope(feed: Feed) -> market_data_pb2.EventEnvelope: result = market_data_pb2.EventEnvelope( schema_name="qdl.marketdata.v2.EventEnvelope", From 84352cab7d5be5cfa716a62e02d82d80d9bd9683 Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Sun, 27 Sep 2026 18:48:56 +0000 Subject: [PATCH 07/17] Diagnose rejected market views against exact canonical lineage --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 25 ++ scripts/probe_exact_market_lineage.py | 314 ++++++++++++++++++++++ tests/test_probe_exact_market_lineage.py | 81 ++++++ 3 files changed, 420 insertions(+) create mode 100644 scripts/probe_exact_market_lineage.py create mode 100644 tests/test_probe_exact_market_lineage.py diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index e1478d84..03cf006a 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -60471,3 +60471,28 @@ Initial builder attempts failed toolchain PATH, then wrong Cargo cache layout; correct existing registry mount is qdl-cargo-home:/usr/local/cargo/registry. No production rollout or test capacity claim from these source/build results. SDK stream/cursor unittest26PASS; combined affected unit count39PASS. + +#### Exact lineage checkpoint - canonical ahead of rejected views + +Status: IN_PROGRESS; EXECUTION_ACCEPTANCE_NOT_FULL remains unchanged. +Read-only180s probe captured5543canonical records and3228Query reads on both +replicas/real live+sandbox identities. BTC/OKX MARK had10rejections: all10 had a +newer same-session/generation canonical pair observed before the request started. +Across the three inspected OKX products,40TRADE stale views also had newer +canonical data before request start. This proves downstream lag for those samples, +not the exact stage responsible; no blanket quiet-market conclusion is valid. +The raw topic returned ACL29; no alternative credential or ACL bypass attempted. +Remaining samples cannot establish provider silence. Canonical inline capture +lineage supports the positive source-arrival findings. Query caches parsed rows +by full canonical bytes, not price-only hash; native projector timing remains +under investigation. No freshness threshold or runtime setting changed. + +Evidence: consumer-realms/evidence/exact-market-lineage-window.jsonl under the +20260927 burst-recovery evidence root; SHA256 +2100d05eef95a8152ef9de5c23f11c56b11586a8291d80fdd0527835a660f99b. +Probe unit6PASS (python3 -m unittest tests.test_probe_exact_market_lineage -v). +Tests cover exact original identity, pre-request ordering, generation mismatch, +incomplete capture, secret exclusion and bounded exclusive evidence files. +Production observation180s:13/13sessionREADY samples; not a burst certificate. +No production mutation; disposable probe clients removed; no image build or +broad cleanup. Published release remainsv2.2.0; candidate branch unchanged. diff --git a/scripts/probe_exact_market_lineage.py b/scripts/probe_exact_market_lineage.py new file mode 100644 index 00000000..4f3d8e69 --- /dev/null +++ b/scripts/probe_exact_market_lineage.py @@ -0,0 +1,314 @@ +#!/usr/bin/env python3 +"""Bounded read-only raw/canonical/exact Query evidence; never an acceptance gate.""" +from __future__ import annotations + +import argparse +import asyncio +import dataclasses +import hashlib +import json +from pathlib import Path +import time + + +VIEW_FIELDS = ("instrument_uid", "instrument_id", "feed", "interval", "quality", + "source", "contract", "observed_at_ns", "received_at_ns", + "watermark_offset", "revision") + + +def exact_view(response): + """Allowlist metadata from the actual response; no payload or signed cursor.""" + row = response.get("data", {}) + return {key: row[key] for key in VIEW_FIELDS if key in row} + + +def trade_state(*, raw_count, canonical_count, query_matches, coverage_complete): + if not coverage_complete: + return "UNASSESSED_INCOMPLETE_CAPTURE" + if raw_count == 0: + return "NO_TRADE_IN_CAPTURE_WINDOW_NOT_SESSION_PROOF" + if canonical_count == 0: + return "RAW_PRESENT_CANONICAL_MISSING_IN_WINDOW" + if not query_matches: + return "CANONICAL_PRESENT_QUERY_JOIN_MISSING" + return "SOURCE_CANONICAL_QUERY_OBSERVED" + + +def exact_join(query, canonical): + """Match original lineage; the public handoff boundary may exceed row offset.""" + view = query.get("exact_view") or {} + event = canonical["event"] + return ( + query["uid"] == event["instrument_uid"] + and query["feed"].lower() == canonical["feed"] + and view.get("watermark_offset", -1) >= canonical["offset"] + and view.get("observed_at_ns") == int(event["source_event_time_ns"]) + and view.get("received_at_ns") == int(event["received_at_ns"]) + and view.get("contract", {}).get("correlation_id") == event["correlation_id"] + ) + + +def newer_before_request(query, selected, candidate): + """Positive lag evidence only: same product/generation, observed before call.""" + old, new = selected["event"], candidate["event"] + return ( + exact_join(query, selected) + and old["instrument_uid"] == new["instrument_uid"] + and selected["feed"] == candidate["feed"] + and selected["partition"] == candidate["partition"] + and candidate["offset"] > selected["offset"] + and candidate["captured_at_ns"] < query["request_started_at_ns"] + and old["source_session_id"] == new["source_session_id"] + and old["connection_generation"] == new["connection_generation"] + and old["authority_revision"] == new["authority_revision"] + and int(new["received_at_ns"]) > int(old["received_at_ns"]) + ) + + +class Evidence: + def __init__(self, path, max_bytes=48 * 1024 * 1024): + self.file = Path(path).open("x") + self.size = 0 + self.max_bytes = max_bytes + self.count = 0 + + def put(self, kind, **row): + line = json.dumps(dict(kind=kind, captured_at_ns=time.time_ns(), **row), + sort_keys=True, separators=(",", ":")) + "\n" + size = len(line.encode()) + if self.size + size > self.max_bytes: + raise RuntimeError("EVIDENCE_BYTE_LIMIT") + self.file.write(line) + self.file.flush() + self.size += size + self.count += 1 + + def close(self): + self.file.close() + + +def raw_metadata(raw): + from qdl.raw.envelope import validate_raw_envelope + validate_raw_envelope(raw) + body = json.loads(raw.raw_frame_bytes) + data = body.get("data", body) + rows = data if isinstance(data, list) else [data] + # Native IDs and clocks only: retain hashes, not provider payloads. + clocks = [{k: item[k] for k in ("ts", "T", "E", "tradeId", "t", "a", "instId", "s") + if k in item} for item in rows if isinstance(item, dict)] + return dict(capture_id=bytes(raw.capture_id).hex(), + raw_frame_sha256=bytes(raw.raw_frame_sha256).hex(), + provider=raw.provider, native_symbol=raw.native_symbol, + native_channel=raw.native_channel, received_at_ns=raw.received_at_ns, + source_session_id=raw.source_session_id, + connection_generation=raw.connection_generation, + authority_revision=raw.authority_revision, + test_provenance=raw.test_provenance, original_clocks=clocks) + + +def canonical_metadata(event): + from google.protobuf.json_format import MessageToDict + from qdl.runtime.mark_index_lineage import paired_mark_index_lineage + result = MessageToDict(event, preserving_proto_field_name=True) + result.pop(event.WhichOneof("payload"), None) + for key in ("event_id", "raw_capture_id", "raw_payload_hash", "canonical_payload_hash"): + result[key] = bytes(getattr(event, key)).hex() + if event.WhichOneof("payload") == "mark_index_price": + try: + pair = dataclasses.asdict(paired_mark_index_lineage(event)) + result["components"] = {k: v.hex() if isinstance(v, bytes) else v + for k, v in pair.items()} + except ValueError: + result["component_lineage"] = "NOT_DERIVED_PAIR" + return result + + +async def run(args): + import yaml + from confluent_kafka import Consumer, KafkaError, TopicPartition + from qdl.marketdata.v2.market_data_pb2 import EventEnvelope + from qdl.provider.v1.raw_provider_pb2 import RawProviderEnvelope + from qdl_sdk.client import AsyncDataLayerClient + from qdl_sdk.credentials import RotatingJwtCredentialProvider + from qdl_sdk.tls import WorkloadTlsConfig + from qdl_sdk.transport import GrpcStreamTransport, RestQueryTransport + from scripts.accept_consumer_realms import requirement + from scripts.kn_canonical_mirror import source_config + + if not 1 <= args.seconds <= 180: + raise ValueError("duration must be 1..180 seconds") + if not 0 <= args.tail_records <= 2000: + raise ValueError("tail must be 0..2000 records per partition") + packet = Path(args.packet) + policy = json.loads((packet / "reader-public/jwt-config.json").read_text()) + catalog = yaml.safe_load(Path(args.catalog).read_text()) + wanted = set(args.uid) + instruments = [r for r in catalog["instruments"] if r["instrument_uid"] in wanted] + if {r["instrument_uid"] for r in instruments} != wanted: + raise ValueError("UID absent from source catalog") + symbols = {r["native_symbol"] for r in instruments} + # OKX's index component is named BTC-USDT, not BTC-USDT-SWAP. + symbols |= {s.removesuffix("-SWAP") for s in symbols} + evidence = Evidence(args.output) + consumer = None + clients = [] + start = time.monotonic() + deadline = start + args.seconds + counters = {"raw": 0, "canonical": 0, "query": 0, "query_refused": 0} + starts, positions, eof = {}, {}, {} + errors = [] + + class ExactTransport(RestQueryTransport): + last = None + + async def snapshot(self, requirement, *, consumer_id): + self.last = None + value = await super().snapshot(requirement, consumer_id=consumer_id) + self.last = exact_view(value) + return value + + try: + evidence.put("scope", schema="qdl.exact-market-lineage.v1", + provenance="production_read_only", test_provenance=False, + instruments=instruments, duration_seconds=args.seconds, + source_catalog_sha256=hashlib.sha256(Path(args.catalog).read_bytes()).hexdigest(), + kafka_commits=False, order_actions=0, + classification_rule="No quiet conclusion from provider_session_state") + config = source_config(argparse.Namespace(source_bootstrap=args.bootstrap, + group_prefix="qdl-r1-reference-parity-", ca=args.ca, + cert=args.cert, key=args.key)) + config["enable.partition.eof"] = True + config["client.id"] = "exact-market-lineage" + consumer = Consumer(config) + assignments = [] + for topic in ("md.raw.realtime.v2", "md.canonical.v2"): + metadata = consumer.list_topics(topic, timeout=5).topics[topic] + if metadata.error: + errors.append(dict(code="TOPIC_METADATA_UNAVAILABLE", topic=topic, kafka_code=metadata.error.code())) + continue + for partition in sorted(metadata.partitions): + tp = TopicPartition(topic, partition) + low, high = consumer.get_watermark_offsets(tp, timeout=5) + offset = max(low, high - args.tail_records) + starts[f"{topic}:{partition}"] = dict(low=low, high=high, start=offset) + assignments.append(TopicPartition(topic, partition, offset)) + consumer.assign(assignments) + evidence.put("kafka_start", partitions=starts, group_id=config["group.id"], + isolation="read_committed", manual_assignment=True) + for realm in ("live", "sandbox"): + cid = f"trading-system.{realm}.stable" + path = packet / "manifests-crypto" / f"{cid}.yaml" + manifest = yaml.safe_load(path.read_text()) + meta = manifest["metadata"] + identity = Path(args.identities) / cid + creds = RotatingJwtCredentialProvider(private_key_file=identity / "private.key", + key_id=cid + "-rs256-v1", algorithm="RS256", issuer=policy["issuer"], + audience=policy["audience"], subject=meta["subject"], environment=realm, + roles=("market_data_reader", "historical_reader", "stream_consumer"), + consumer_manifest_revision=meta["revision"], lifetime_seconds=300, + refresh_before_seconds=30) + tls = WorkloadTlsConfig(identity / "ca.crt", identity / "client.crt", identity / "client.key") + selected = [requirement(r) for r in manifest["spec"]["requirements"] + if r["instrument_uid"] in wanted and r["feed"] in {"TRADE", "MARK_INDEX_PRICE"}] + for replica in (1, 2): + transport = ExactTransport(f"https://query_v2_{replica}:8200", + credential_provider=creds, tls=tls, timeout_seconds=3) + stream = GrpcStreamTransport("unused.invalid:8210", credential_provider=creds, tls=tls) + client = AsyncDataLayerClient(query_transport=transport, stream_transport=stream, consumer_id=cid) + clients.append((cid, replica, selected, transport, client)) + + async def queries(): + while time.monotonic() < deadline: + for cid, replica, selected, transport, client in clients: + for req in selected: + remaining = deadline - time.monotonic() + if remaining <= 0: + return + row = dict(consumer_id=cid, replica=replica, uid=req.instrument_uid, + feed=req.feed.value, request_started_at_ns=time.time_ns()) + try: + async with asyncio.timeout(min(3.5, remaining)): + await client.snapshot(req) + row["status"] = "TYPED_RESPONSE" + except Exception as error: + row.update(status="REFUSED", code=getattr(error, "code", type(error).__name__), + diagnostics=getattr(error, "diagnostics", None)) + counters["query_refused"] += 1 + row["exact_view"] = transport.last + evidence.put("query", **row) + counters["query"] += 1 + await asyncio.sleep(.025) + await asyncio.sleep(.5) + + async def kafka(): + while time.monotonic() < deadline: + # Bounded microbatches yield to Query; no unbounded queue/list. + for _ in range(100): + msg = consumer.poll(0) + if msg is None: + break + key = f"{msg.topic()}:{msg.partition()}" + if msg.error(): + if msg.error().code() == KafkaError._PARTITION_EOF: + positions[key] = msg.offset() + eof[key] = dict(offset=msg.offset(), observed_at_ns=time.time_ns()) + continue + errors.append(dict(code=msg.error().code(), topic=msg.topic())) + raise RuntimeError("KAFKA_READ_ERROR") + positions[key] = msg.offset() + 1 + where = dict(topic=msg.topic(), partition=msg.partition(), offset=msg.offset(), + kafka_timestamp_ms=msg.timestamp()[1]) + if msg.topic() == "md.raw.realtime.v2": + raw = RawProviderEnvelope.FromString(msg.value()) + if raw.native_symbol not in symbols or not any( + word in raw.native_channel.lower() for word in ("trade", "mark", "index")): + continue + evidence.put("raw", **where, **raw_metadata(raw)) + counters["raw"] += 1 + else: + event = EventEnvelope.FromString(msg.value()) + if event.instrument_uid not in wanted or event.WhichOneof("payload") not in {"trade", "mark_index_price"}: + continue + record = canonical_metadata(event) + headers = dict(msg.headers() or ()) + inline = headers.get("qdl-raw-provider-envelope") + if inline: + record["inline_raw"] = raw_metadata(RawProviderEnvelope.FromString(inline)) + evidence.put("canonical", **where, feed=event.WhichOneof("payload"), event=record) + counters["canonical"] += 1 + await asyncio.sleep(.01) + + tasks = [asyncio.create_task(queries()), asyncio.create_task(kafka())] + try: + await asyncio.gather(*tasks) + finally: + for task in tasks: + task.cancel() + await asyncio.gather(*tasks, return_exceptions=True) + except Exception as error: + errors.append(dict(code=type(error).__name__)) + finally: + for _, _, _, _, client in clients: + await client.close() + if consumer is not None: + consumer.close() + try: + evidence.put("summary", counts=counters, errors=errors, start_offsets=starts, + next_positions=positions, last_partition_eof=eof, elapsed_seconds=time.monotonic() - start, + historical_receipt_reconstructed=False, + quiet_certified=False, production_writes=0) + finally: + evidence.close() + print(json.dumps(dict(counts=counters, errors=errors, evidence_bytes=evidence.size))) + return not errors + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + for name in ("packet", "identities", "catalog", "output", "ca", "cert", "key"): + parser.add_argument("--" + name, required=True) + parser.add_argument("--uid", action="append", required=True) + parser.add_argument("--seconds", type=int, default=180) + parser.add_argument("--tail-records", type=int, default=400) + parser.add_argument("--bootstrap", default="kafka1:9092,kafka2:9092,kafka3:9092") + raise SystemExit(0 if asyncio.run(run(parser.parse_args())) else 1) diff --git a/tests/test_probe_exact_market_lineage.py b/tests/test_probe_exact_market_lineage.py new file mode 100644 index 00000000..dcb314e9 --- /dev/null +++ b/tests/test_probe_exact_market_lineage.py @@ -0,0 +1,81 @@ +"""Synthetic test-only fixtures; not provider acceptance evidence.""" +import json +from pathlib import Path +import tempfile +import unittest + +from scripts.probe_exact_market_lineage import Evidence, exact_view, trade_state, exact_join, newer_before_request + + +class ExactMarketLineageTests(unittest.TestCase): + def test_exact_view_preserves_flags_and_original_clocks_without_secrets(self): + row = {"quality": {"flags": ["INDEX_STALE"], "execution_eligible": False}, + "observed_at_ns": 123, "received_at_ns": 124, + "watermark_offset": 9, "contract": {"connection_generation": 7}, + "cursor": "secret", "payload": {"price": "42"}} + result = exact_view({"data": row, "cursor": "secret"}) + self.assertEqual(result["quality"], row["quality"]) + self.assertEqual(result["observed_at_ns"], 123) + self.assertEqual(result["contract"], row["contract"]) + self.assertNotIn("secret", json.dumps(result)) + self.assertNotIn("payload", result) + + def test_missing_raw_never_claims_quiet_from_session(self): + self.assertEqual(trade_state(raw_count=0, canonical_count=0, + query_matches=False, coverage_complete=True), + "NO_TRADE_IN_CAPTURE_WINDOW_NOT_SESSION_PROOF") + + def test_incomplete_capture_fails_closed(self): + self.assertEqual(trade_state(raw_count=12, canonical_count=12, + query_matches=True, coverage_complete=False), "UNASSESSED_INCOMPLETE_CAPTURE") + + def test_distinguish_pipeline_stages(self): + self.assertEqual(trade_state(raw_count=1, canonical_count=0, + query_matches=False, coverage_complete=True), "RAW_PRESENT_CANONICAL_MISSING_IN_WINDOW") + self.assertEqual(trade_state(raw_count=1, canonical_count=1, + query_matches=False, coverage_complete=True), "CANONICAL_PRESENT_QUERY_JOIN_MISSING") + self.assertEqual(trade_state(raw_count=1, canonical_count=1, + query_matches=True, coverage_complete=True), "SOURCE_CANONICAL_QUERY_OBSERVED") + + def test_join_and_positive_lag_require_original_identity_and_pre_request_capture(self): + import copy + event = dict(instrument_uid="test-only", source_event_time_ns="10", + received_at_ns="11", correlation_id="capture", source_session_id="s", + connection_generation="7", authority_revision="1") + selected = dict(event=event, feed="trade", partition=2, offset=4, captured_at_ns=20) + query = dict(uid="test-only", feed="TRADE", request_started_at_ns=30, + exact_view=dict(observed_at_ns=10, received_at_ns=11, watermark_offset=4, + contract=dict(correlation_id="capture"))) + newer = copy.deepcopy(selected) + newer.update(offset=5, captured_at_ns=29) + newer["event"]["received_at_ns"] = "12" + self.assertTrue(exact_join(query, selected)) + query["exact_view"]["watermark_offset"] = 8 + self.assertTrue(exact_join(query, selected)) + query["exact_view"]["watermark_offset"] = 3 + self.assertFalse(exact_join(query, selected)) + query["exact_view"]["watermark_offset"] = 4 + self.assertTrue(newer_before_request(query, selected, newer)) + newer["captured_at_ns"] = 31 + self.assertFalse(newer_before_request(query, selected, newer)) + newer["captured_at_ns"] = 29 + newer["event"]["connection_generation"] = "8" + self.assertFalse(newer_before_request(query, selected, newer)) + query["exact_view"]["contract"]["correlation_id"] = "different" + self.assertFalse(exact_join(query, selected)) + + def test_bounded_evidence_exclusive_and_no_overwrite(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "test-only.jsonl" + writer = Evidence(path, max_bytes=200) + writer.put("test", test_provenance=True) + with self.assertRaises(RuntimeError): + writer.put("test", value="x" * 200) + writer.close() + self.assertLessEqual(path.stat().st_size, 200) + with self.assertRaises(FileExistsError): + Evidence(path) + + +if __name__ == "__main__": + unittest.main() From b25c136fd73dfb3c7a640b2c3c7688ca37a7064c Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Sun, 27 Sep 2026 19:08:24 +0000 Subject: [PATCH 08/17] Separate execution reference acceptance from generic mark snapshots --- DATA_LAYER_SERVICE_ACCESS_GUIDE.md | 16 ++ DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 25 ++ scripts/accept_consumer_realms.py | 44 ++- scripts/probe_projector_stage_timing.py | 295 ++++++++++++++++++++ tests/test_consumer_realm_execution_path.py | 42 +++ tests/test_probe_projector_stage_timing.py | 46 +++ 6 files changed, 464 insertions(+), 4 deletions(-) create mode 100644 scripts/probe_projector_stage_timing.py create mode 100644 tests/test_consumer_realm_execution_path.py create mode 100644 tests/test_probe_projector_stage_timing.py diff --git a/DATA_LAYER_SERVICE_ACCESS_GUIDE.md b/DATA_LAYER_SERVICE_ACCESS_GUIDE.md index 7f78821f..3afac45d 100644 --- a/DATA_LAYER_SERVICE_ACCESS_GUIDE.md +++ b/DATA_LAYER_SERVICE_ACCESS_GUIDE.md @@ -99,6 +99,22 @@ execution or risk authority except a separately declared fresh `MARK_INDEX_PRICE` read. A new alpha must add its own manifest requirement before calling a reference metric; it must not reuse another alpha's identity. +### Execution MARK/INDEX Read Contract + +Trading System's `latest_execution_feed(..., MARK_INDEX_PRICE)` uses +`POST /v2/market-data/reference:batch` with EXECUTION grade, one complete row, +`limit=page_size=max_pages=1`, and the identity's signed freshness/session policy. +The Kafka-native backend reads its verified market cache; this is not venue REST. +Component/session evidence may admit unchanged values only within the signed +component cadence. Disconnect, gap, generation/config mismatch or expired +component evidence remains blocked. Original source timestamps are preserved. + +Generic `snapshot(MARK_INDEX_PRICE)` is a different read contract. Its event-age +rejection does not establish a failure of the component-aware execution reference +path. Tests must name the endpoint they actually exercised and retain both +outcomes; never silently substitute one to turn a failing gate green. TRADE +session liveness likewise never makes an old last-trade price execution-eligible. + ## 1. Connection Architecture Production services should **never** connect directly to external exchanges (Binance, DNSE, etc.) if they are running within the `bobby_network`. Instead, they must use the `data_layer` as a unified gateway. diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index 03cf006a..f6daa4bb 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -60496,3 +60496,28 @@ incomplete capture, secret exclusion and bounded exclusive evidence files. Production observation180s:13/13sessionREADY samples; not a burst certificate. No production mutation; disposable probe clients removed; no image build or broad cleanup. Published release remainsv2.2.0; candidate branch unchanged. + +#### MARK endpoint semantics and native state timing checkpoint + +Status IN_PROGRESS, not FULL. Read-only exact-hash canonical-to-Redis observer +interval553samples: median361.184ms,p95589.722ms,max743.927ms. These are observer +intervals, not broker commit timestamps. State-topic ACL29 prevents attributing +A commit versus state visibility versus B apply; no bypass or tuning performed. +TS-shaped MARK reference:batch658/658itemOK. Adjacent generic snapshot2/658refused +at2251/2275ms while reference accepted the same watermark with signed component +cadence/session evidence. The earlier generic SDK SOURCE_NON_AUTHORITATIVE must +not be reported as rejection of the TS reference path or loss of source authority. +Keep strict generic snapshot behavior and independently report both paths. +Acceptance/guide correction scope: explicitly exercise execution reference batch +with the same manifest requirements; retain generic refusal evidence, no silent +replacement of a required endpoint or weakening freshness. Add focused tests. +Timing probe5unitPASS; evidence projector-stage-timing-window.jsonl SHA256 +2635ff41265720c8e3156299f915b3b9f9fe3f5105bd2fad6bb6646745156dbe. +Runtime unchanged; scoped client removed, no build or broad cleanup. + +Acceptance now records an additional execution reference path for each declared +EXECUTION MARK requirement, preserving generic snapshot results and all existing +gates. Reference item errors stay visible with require_all=False as in TS; +status must beOK, and no retry or quiet-price exemption is introduced. +Guide clarifies the two contracts.8focused unitPASS (3path+5timing), existing +Query image with source mounted read-only/network none. No runtime rollout. diff --git a/scripts/accept_consumer_realms.py b/scripts/accept_consumer_realms.py index f0c5d85b..93c5eedc 100644 --- a/scripts/accept_consumer_realms.py +++ b/scripts/accept_consumer_realms.py @@ -11,6 +11,7 @@ from qdl_sdk.client import AsyncDataLayerClient from qdl_sdk.credentials import RotatingJwtCredentialProvider from qdl_sdk.errors import DataLayerError +from qdl_sdk.reference import ReferenceRequirement, ReferenceProduct from qdl_sdk.models import GapPolicy, RecoveryPolicy, BarRevisionPolicy from qdl_sdk.tls import WorkloadTlsConfig from qdl_sdk.transport import RestQueryTransport, GrpcStreamTransport @@ -30,10 +31,30 @@ def requirement(row): bar_revision_policy=BarRevisionPolicy(row.get("bar_revision_policy", "LATEST"))) +def execution_reference(requirement): + if requirement.feed is not Feed.MARK_INDEX_PRICE or requirement.consumer_grade is not Grade.EXECUTION: + raise ValueError("execution MARK_INDEX_PRICE requirement required") + return ReferenceRequirement( + instrument_uid=requirement.instrument_uid, product=ReferenceProduct.MARK_INDEX_PRICE, + consumer_grade=Grade.EXECUTION, source_policy_id=requirement.source_policy_id, + limit=1, page_size=1, max_pages=1, require_full_coverage=True, + max_freshness_ms=requirement.max_freshness_ms, + event_recency_policy=requirement.event_recency_policy, + max_session_liveness_ms=requirement.max_session_liveness_ms, + ) + + +async def read_execution_reference(client, requirement): + # Same one-item path as TS; item errors remain visible, never exempted. + response = await client.reference_batch((execution_reference(requirement),), require_all=False) + from scripts.probe_projector_stage_timing import reference_metadata + return reference_metadata(response.model_dump(mode="json")) + + async def run(args): packet = Path(args.packet) policy = json.loads((packet / "reader-public/jwt-config.json").read_text()) - reads, negatives, streams = [], [], [] + reads, negatives, streams, reference_reads = [], [], [], [] for path in sorted((packet / "manifests-crypto").glob("*.yaml")): payload = yaml.safe_load(path.read_text()) meta, declared = payload["metadata"], payload["spec"]["requirements"] @@ -86,6 +107,20 @@ def credential(**overrides): row.update(status="REFUSED", code=getattr(error, "code", type(error).__name__), detail=str(error)[:180], diagnostics=getattr(error, "diagnostics", None)) row["call_to_result_ms"] = (time.perf_counter() - start) * 1000 reads.append(row) + if req.feed is Feed.MARK_INDEX_PRICE and req.consumer_grade is Grade.EXECUTION: + reference_row = dict(consumer_id=cid, realm=realm, replica=replica, + instrument_uid=req.instrument_uid, endpoint="reference:batch") + reference_start = time.perf_counter() + try: + evidence = await read_execution_reference(client, req) + reference_row.update(status="TYPED_RESPONSE", evidence=evidence, + item_ok=len(evidence["results"]) == 1 and evidence["results"][0]["status"] == "OK") + except Exception as error: + reference_row.update(status="REFUSED", item_ok=False, + code=getattr(error, "code", type(error).__name__), + diagnostics=getattr(error, "diagnostics", None)) + reference_row["call_to_result_ms"] = (time.perf_counter() - reference_start) * 1000 + reference_reads.append(reference_row) await asyncio.sleep(.1) probe = requirement(next(r for r in selected if r["feed"] == "TRADE")) for name, kwargs, consumer in ( @@ -119,15 +154,16 @@ def credential(**overrides): failures = [r for r in reads if r["status"] != "TYPED_RESPONSE"] usable = [r for r in reads if r.get("execution_eligible") is True] report = dict(schema="qdl.consumer-realm-acceptance.v1", order_actions=0, - snapshots=reads, negatives=negatives, streams=streams, + snapshots=reads, negatives=negatives, streams=streams, execution_references=reference_reads, + execution_reference_gate=bool(reference_reads) and all(r["item_ok"] for r in reference_reads), typed_response_gate=bool(reads) and not failures, execution_usable_count=len(usable), total_reads=len(reads), negative_gate=bool(negatives) and all(r["refused"] for r in negatives), stream_gate=len(streams) == 12 and all(r["status"] == "STREAM_ACK_PASS" for r in streams)) report["all_reads_execution_eligible"] = bool(reads) and len(usable) == len(reads) Path(args.output).write_text(json.dumps(report, indent=2)) - print(json.dumps({k:v for k,v in report.items() if k not in {"snapshots", "negatives", "streams"}})) - return report["typed_response_gate"] and report["negative_gate"] and report["stream_gate"] + print(json.dumps({k:v for k,v in report.items() if k not in {"snapshots", "negatives", "streams", "execution_references"}})) + return report["typed_response_gate"] and report["negative_gate"] and report["stream_gate"] and report["execution_reference_gate"] if __name__ == "__main__": diff --git a/scripts/probe_projector_stage_timing.py b/scripts/probe_projector_stage_timing.py new file mode 100644 index 00000000..bcd1df0e --- /dev/null +++ b/scripts/probe_projector_stage_timing.py @@ -0,0 +1,295 @@ +#!/usr/bin/env python3 +"""Read-only bounded canonical/state/Redis timing; never change runtime tuning.""" +from __future__ import annotations + +import argparse +import asyncio +import hashlib +import json +from pathlib import Path +import time + +from scripts.probe_exact_market_lineage import Evidence + + +def reference_metadata(response): + results = [] + for item in response.get("results", []): + out = {k: item[k] for k in ("instrument_uid", "product", "status", "problem") if k in item} + data = item.get("data") + if data: + out["data"] = {k: data[k] for k in ("status", "received_at_ns", "coverage", "lineage", + "error_code", "error_detail", "cache_hit", "coalesced", "quality") if k in data} + allowed = {"native_symbol", "execution_view", "freshness_basis", "source_event_time_ns", + "provider_confirmation_ns", "connection_generation", "gateway_lease_epoch", "delivery_stage", + "spool_watermark_offset", "event_recency_policy", "recency_mode", "provider_session_state", + "provider_session_liveness_ms", "provider_session_checked_at_ns", + "component_mark_received_at_ns", "component_index_received_at_ns", + "component_mark_quiet_after_ms", "component_index_quiet_after_ms"} + out["data"]["observations"] = [dict(observed_at_ns=o.get("observed_at_ns"), + labels={k:v for k,v in o.get("labels", {}).items() if k in allowed}, + field_names=[f["name"] for f in o.get("fields", [])]) for o in data.get("observations", [])] + results.append(out) + return dict(partial=response.get("partial"), results=results) + + +async def query_probe(args, evidence, deadline): + import yaml + from scripts.accept_consumer_realms import requirement + from scripts.probe_exact_market_lineage import exact_view + from qdl_sdk.client import AsyncDataLayerClient + from qdl_sdk.credentials import RotatingJwtCredentialProvider + from qdl_sdk.reference import ReferenceRequirement, ReferenceProduct + from qdl_sdk.models import Grade + from qdl_sdk.tls import WorkloadTlsConfig + from qdl_sdk.transport import RestQueryTransport, GrpcStreamTransport + packet = Path(args.packet) + cid = "trading-system.live.stable" + policy = json.loads((packet / "reader-public/jwt-config.json").read_text()) + manifest = yaml.safe_load((packet / "manifests-crypto" / (cid + ".yaml")).read_text()) + req = requirement(next(r for r in manifest["spec"]["requirements"] + if r["instrument_uid"] == args.uid and r["feed"] == "MARK_INDEX_PRICE")) + ref = ReferenceRequirement(instrument_uid=req.instrument_uid, product=ReferenceProduct.MARK_INDEX_PRICE, + consumer_grade=Grade.EXECUTION, source_policy_id=req.source_policy_id, limit=1, page_size=1, max_pages=1, + max_freshness_ms=req.max_freshness_ms, event_recency_policy=req.event_recency_policy, + max_session_liveness_ms=req.max_session_liveness_ms, require_full_coverage=True) + evidence.put("query_scope", reference_requirement=ref.model_dump(mode="json"), require_all=False, + generic_snapshot_is_not_ts_reference_path=True) + identity_path = Path(args.identity) + creds = RotatingJwtCredentialProvider(private_key_file=identity_path / "private.key", + key_id=cid + "-rs256-v1", algorithm="RS256", issuer=policy["issuer"], audience=policy["audience"], + subject=manifest["metadata"]["subject"], environment="live", + roles=("market_data_reader", "historical_reader", "stream_consumer"), + consumer_manifest_revision=manifest["metadata"]["revision"], lifetime_seconds=300, refresh_before_seconds=30) + tls = WorkloadTlsConfig(identity_path / "ca.crt", identity_path / "client.crt", identity_path / "client.key") + class ExactTransport(RestQueryTransport): + last = None + async def snapshot(self, requirement, *, consumer_id): + self.last = None + value = await super().snapshot(requirement, consumer_id=consumer_id) + self.last = exact_view(value) + return value + async def reference_batch(self, requirements, *, consumer_id, require_all): + self.last = None + value = await super().reference_batch(requirements, consumer_id=consumer_id, require_all=require_all) + self.last = reference_metadata(value) + return value + clients = [] + try: + for replica in (1, 2): + transport = ExactTransport(f"https://query_v2_{replica}:8200", credential_provider=creds, tls=tls, timeout_seconds=2) + stream = GrpcStreamTransport("unused.invalid:8210", credential_provider=creds, tls=tls) + clients.append((replica, transport, AsyncDataLayerClient(query_transport=transport, stream_transport=stream, consumer_id=cid))) + pair_id = 0 + while time.monotonic() < deadline: + pair_id += 1 + for replica, transport, client in clients: + for path in ("generic_snapshot", "ts_reference_batch"): + remaining = deadline - time.monotonic() + if remaining <= 0: + return + row = dict(pair_id=pair_id, replica=replica, path=path, request_started_at_ns=time.time_ns()) + try: + async with asyncio.timeout(min(remaining, 2.5)): + if path == "generic_snapshot": + await client.snapshot(req) + else: + await client.reference_batch((ref,), require_all=False) + row["status"] = "TYPED_RESPONSE" + except Exception as error: + row.update(status="REFUSED", code=getattr(error, "code", type(error).__name__), diagnostics=getattr(error, "diagnostics", None)) + evidence.put("query", **row, response_at_ns=time.time_ns(), exact_view=transport.last) + await asyncio.sleep(.5) + finally: + for _, _, client in clients: + await client.close() + + +def classify_stage(canonical_ns, state_ns, redis_ns): + if state_ns is None: + return "CANONICAL_TO_REDIS_ONLY_STATE_UNAVAILABLE" + if state_ns < canonical_ns or redis_ns < state_ns: + return "OBSERVER_ORDER_INVERSION_NOT_NEGATIVE_LATENCY" + return "OBSERVED_STAGE_INTERVALS_NOT_BROKER_COMMIT_TIMES" + + +def counter_rate(before, after, name): + elapsed = after["at_ms"] - before["at_ms"] + delta = after["stage_a"][name] - before["stage_a"][name] + if elapsed <= 0 or delta < 0 or before["started_ms"] != after["started_ms"]: + return None + return delta * 1000 / elapsed + + +def identity(event, payload): + return dict(uid=event.instrument_uid, event_id=bytes(event.event_id).hex(), + canonical_sha256=hashlib.sha256(payload).hexdigest(), + source_event_time_ns=event.source_event_time_ns, + received_at_ns=event.received_at_ns, + source_session_id=event.source_session_id, + connection_generation=event.connection_generation, + correlation_id=event.correlation_id) + + +async def run(args): + from confluent_kafka import Consumer, KafkaError, TopicPartition + import redis.asyncio as redis + from qdl.marketdata.v2.market_data_pb2 import EventEnvelope + from qdl.projection.kn_state_codec import decode_frame, decode_latest_value, state_partition + from qdl.projection.state_contract import LogicalProductKey + from scripts.kn_canonical_mirror import source_config + + if not 1 <= args.seconds <= 180: + raise ValueError("duration must be 1..180 seconds") + lpk = LogicalProductKey("paper", "OKX", "SWAP", args.uid, "MARK_INDEX_PRICE") + product = lpk.encode() + partition = state_partition(lpk, 6) + prefix = "kn3:paper:" + pointer = prefix + "ptr:" + product + ckpt = prefix + "ckpt:md.latest.v2:" + str(partition) + evidence = Evidence(args.output, max_bytes=24 * 1024 * 1024) + config = source_config(argparse.Namespace(source_bootstrap=args.bootstrap, + group_prefix="qdl-r1-reference-parity-", ca=args.ca, cert=args.cert, key=args.key)) + # These settings affect only this observer, never a native runtime consumer. + config.update({"fetch.wait.max.ms": 20, "fetch.queue.backoff.ms": 10, + "enable.partition.eof": True, "client.id": "projector-stage-observer"}) + consumer = Consumer(config) + cache = redis.Redis.from_url(args.redis_url, socket_timeout=3, socket_connect_timeout=3) + errors = [] + counts = {"canonical": 0, "state": 0, "redis": 0, "status": 0} + started = time.monotonic() + deadline = started + args.seconds + eof = {} + try: + evidence.put("scope", schema="qdl.projector-stage-timing.v1", product=product, + duration_seconds=args.seconds, test_provenance=False, production_writes=0, + kafka_commits=False, state_partition=partition, + script_sha256=hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), + observer_fetch_wait_ms=20, observer_fetch_queue_backoff_ms=10, + timing_semantics="local observations, not exact broker commit/apply timestamps") + assignments = [] + for topic in ("md.canonical.v2", "md.latest.v2"): + metadata = consumer.list_topics(topic, timeout=5).topics[topic] + if metadata.error: + error = dict(code="TOPIC_AUTHORIZATION_FAILED" if metadata.error.code() == 29 else "TOPIC_UNAVAILABLE", + kafka_code=metadata.error.code(), topic=topic) + errors.append(error) + evidence.put("access", **error) + continue + partitions = sorted(metadata.partitions) if topic == "md.canonical.v2" else [partition] + for p in partitions: + low, high = consumer.get_watermark_offsets(TopicPartition(topic, p), timeout=5) + offset = max(low, high - 100) + assignments.append(TopicPartition(topic, p, offset)) + evidence.put("start", topic=topic, partition=p, low=low, high=high, offset=offset) + consumer.assign(assignments) + + async def kafka_loop(): + while time.monotonic() < deadline: + for _ in range(100): + msg = consumer.poll(0) + if msg is None: + break + if msg.error(): + if msg.error().code() == KafkaError._PARTITION_EOF: + eof[f"{msg.topic()}:{msg.partition()}"] = dict(offset=msg.offset(), observed_at_ns=time.time_ns()) + continue + raise RuntimeError("KAFKA_READ_ERROR") + observed = time.time_ns() + if msg.topic() == "md.canonical.v2": + body = msg.value() + event = EventEnvelope.FromString(body) + if event.instrument_uid != args.uid or event.WhichOneof("payload") != "mark_index_price": + continue + kind = "canonical" + source = dict(partition=msg.partition(), offset=msg.offset()) + else: + if msg.key() != product.encode() or not msg.value(): + continue + frame = decode_frame(msg.value()) + body = frame.envelope + event = EventEnvelope.FromString(body) + source = dict(topic_id=frame.source.topic_id, partition=frame.source.partition, + offset=frame.source.offset) + kind = "state" + evidence.put(kind, observed_at_ns=observed, topic=msg.topic(), + partition=msg.partition(), offset=msg.offset(), source=source, + kafka_timestamp_type=msg.timestamp()[0], kafka_timestamp_ms=msg.timestamp()[1], + **identity(event, body)) + counts[kind] += 1 + await asyncio.sleep(.005) + + async def redis_loop(): + while time.monotonic() < deadline: + begin = time.time_ns() + ready = await cache.hmget(pointer, "ready", "fence") + if ready[0] is None: + raise RuntimeError("REDIS_NO_GENERATION") + key = prefix + "l:" + ready[0].decode() + ":" + product + async with cache.pipeline(transaction=False) as pipe: + pipe.hmget(key, "v", "t", "p", "o") + pipe.hmget(pointer, "ready", "fence") + pipe.hmget(ckpt, "next", "at_ms", "fence") + values, after, checkpoint = await pipe.execute() + end = time.time_ns() + if ready != after: + evidence.put("redis_generation_changed", request_started_at_ns=begin) + continue + if values[0] is None: + raise RuntimeError("REDIS_LATEST_MISSING") + decoded = decode_latest_value(values[0]) + if decoded.source_offset != int(values[3]): + raise RuntimeError("REDIS_SOURCE_OFFSET_MISMATCH") + event = EventEnvelope.FromString(decoded.canonical) + evidence.put("redis", request_started_at_ns=begin, response_at_ns=end, + generation=int(ready[0]), fence=int(ready[1] or 0), + source=dict(topic_id=values[1].decode(), partition=int(values[2]), offset=int(values[3])), + state_checkpoint_next_offset=int(checkpoint[0]) if checkpoint[0] else None, + state_checkpoint_at_ms=int(checkpoint[1]) if checkpoint[1] else None, + state_checkpoint_fence=int(checkpoint[2]) if checkpoint[2] else None, + **identity(event, decoded.canonical)) + counts["redis"] += 1 + await asyncio.sleep(.025) + + async def status_loop(): + previous = {} + while time.monotonic() < deadline: + for i, path in enumerate(args.status): + status = json.loads(Path(path).read_text()) + if status["at_ms"] != previous.get(i): + evidence.put("status", replica=i + 1, status=status) + counts["status"] += 1 + previous[i] = status["at_ms"] + await asyncio.sleep(.5) + + tasks = [asyncio.create_task(query_probe(args, evidence, deadline))] + tasks.extend(asyncio.create_task(fn()) for fn in (kafka_loop, redis_loop, status_loop)) + try: + await asyncio.gather(*tasks) + finally: + for task in tasks: + task.cancel() + await asyncio.gather(*tasks, return_exceptions=True) + except Exception as error: + errors.append(dict(code=type(error).__name__)) + finally: + consumer.close() + await cache.aclose() + evidence.put("summary", counts=counts, errors=errors, last_eof=eof, + elapsed_seconds=time.monotonic() - started, production_writes=0, + stage_split_proven=False) + evidence.close() + print(json.dumps(dict(counts=counts, errors=errors))) + return not errors + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + for name in ("ca", "cert", "key", "output", "packet", "identity"): + parser.add_argument("--" + name, required=True) + parser.add_argument("--status", action="append", default=[]) + parser.add_argument("--seconds", type=int, default=180) + parser.add_argument("--uid", default="fb26214c-7b9b-5961-95b2-55154755af0f") + parser.add_argument("--redis-url", default="redis://market_cache:6379/0") + parser.add_argument("--bootstrap", default="kafka1:9092,kafka2:9092,kafka3:9092") + raise SystemExit(0 if asyncio.run(run(parser.parse_args())) else 1) diff --git a/tests/test_consumer_realm_execution_path.py b/tests/test_consumer_realm_execution_path.py new file mode 100644 index 00000000..21cbb735 --- /dev/null +++ b/tests/test_consumer_realm_execution_path.py @@ -0,0 +1,42 @@ +import unittest +from types import SimpleNamespace +from unittest.mock import AsyncMock + +from qdl_sdk import DataRequirement, Feed, Grade, StalePolicy +from scripts.accept_consumer_realms import execution_reference, read_execution_reference + + +class ExecutionPathTests(unittest.IsolatedAsyncioTestCase): + def requirement(self, feed=Feed.MARK_INDEX_PRICE): + return DataRequirement(instrument_uid="test-only", feed=feed, + consumer_grade=Grade.EXECUTION, source_policy_id="test-policy", + max_freshness_ms=2000, event_recency_policy=StalePolicy.OBSERVE, + max_session_liveness_ms=2000) + + def test_ts_reference_contract_preserves_signed_limits(self): + ref = execution_reference(self.requirement()) + self.assertEqual((ref.limit, ref.page_size, ref.max_pages), (1, 1, 1)) + self.assertTrue(ref.require_full_coverage) + self.assertEqual(ref.max_freshness_ms, 2000) + self.assertEqual(ref.max_session_liveness_ms, 2000) + self.assertEqual(ref.event_recency_policy, StalePolicy.OBSERVE) + self.assertEqual(ref.source_policy_id, "test-policy") + + def test_trade_is_not_silently_replaced(self): + with self.assertRaises(ValueError): + execution_reference(self.requirement(Feed.TRADE)) + + async def test_item_refusal_is_preserved_not_accepted(self): + payload = {"partial": True, "results": [{"instrument_uid": "test-only", + "product": "MARK_INDEX_PRICE", "status": "ERROR", + "problem": {"code": "DATA_STALE"}}]} + response = SimpleNamespace(model_dump=lambda **kw: payload) + client = SimpleNamespace(reference_batch=AsyncMock(return_value=response)) + evidence = await read_execution_reference(client, self.requirement()) + self.assertEqual(evidence, payload) + self.assertIs(client.reference_batch.call_args.kwargs["require_all"], False) + self.assertEqual(client.reference_batch.call_count, 1) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_probe_projector_stage_timing.py b/tests/test_probe_projector_stage_timing.py new file mode 100644 index 00000000..e601d078 --- /dev/null +++ b/tests/test_probe_projector_stage_timing.py @@ -0,0 +1,46 @@ +"""Synthetic unit evidence only, not production latency evidence.""" +import unittest + +from scripts.probe_projector_stage_timing import classify_stage, counter_rate, reference_metadata + + +class StageTimingTests(unittest.TestCase): + def test_reference_preserves_problem_components_without_price_or_secrets(self): + import json + response = {"partial": True, "results": [{"status": "ERROR", "problem": {"code": "DATA_STALE"}}, + {"status": "OK", "data": {"received_at_ns": 5, "observations": [{"observed_at_ns": 4, + "fields": [{"name": "mark_price", "value": "SECRET_PRICE"}], + "labels": {"component_mark_received_at_ns": "3", "recency_mode": "COMPONENT_SESSION_LIVE", + "cursor": "SECRET_CURSOR"}}]}}]} + result = reference_metadata(response) + self.assertEqual(result["results"][0]["problem"], {"code": "DATA_STALE"}) + self.assertEqual(result["results"][1]["data"]["observations"][0]["labels"]["component_mark_received_at_ns"], "3") + self.assertNotIn("SECRET", json.dumps(result)) + + def test_missing_state_never_localizes_stage(self): + self.assertEqual(classify_stage(10, None, 30), + "CANONICAL_TO_REDIS_ONLY_STATE_UNAVAILABLE") + + def test_observer_order_is_not_negative_pipeline_latency(self): + self.assertEqual(classify_stage(20, 10, 30), + "OBSERVER_ORDER_INVERSION_NOT_NEGATIVE_LATENCY") + self.assertEqual(classify_stage(10, 30, 20), + "OBSERVER_ORDER_INVERSION_NOT_NEGATIVE_LATENCY") + + def test_ordered_observations_are_not_broker_commit_times(self): + self.assertEqual(classify_stage(10, 20, 30), + "OBSERVED_STAGE_INTERVALS_NOT_BROKER_COMMIT_TIMES") + + def test_counter_rates_reject_restarts_and_resets(self): + first = dict(at_ms=1000, started_ms=1, stage_a=dict(transactions=10)) + last = dict(at_ms=6000, started_ms=1, stage_a=dict(transactions=35)) + self.assertEqual(counter_rate(first, last, "transactions"), 5) + last["started_ms"] = 2 + self.assertIsNone(counter_rate(first, last, "transactions")) + last["started_ms"] = 1 + last["stage_a"]["transactions"] = 9 + self.assertIsNone(counter_rate(first, last, "transactions")) + + +if __name__ == "__main__": + unittest.main() From 6f9bb30e88dc6d8d28c18bd8a1227a2f9acb596b Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Sun, 27 Sep 2026 19:10:42 +0000 Subject: [PATCH 09/17] Align captured trade fixture with production delivery contract --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 26 +++++++++++++++++++ .../examples/kn_trade_burst_gateway.rs | 23 ++++++++++++++-- 2 files changed, 47 insertions(+), 2 deletions(-) diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index f6daa4bb..511cc3fb 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -60521,3 +60521,29 @@ gates. Reference item errors stay visible with require_all=False as in TS; status must beOK, and no retry or quiet-price exemption is introduced. Guide clarifies the two contracts.8focused unitPASS (3path+5timing), existing Query image with source mounted read-only/network none. No runtime rollout. + +#### Actual adapter capacity checkpoint - no sustained capacity pass + +Native driver fixture corrected require_final_bars=false to match actual TS +TRADE signed digest;2Rust tests/release buildPASS. TS actual-facade harness and +focused regressions169PASS. Core production conversion is no longer replaced by +a lookalike test loop. Exact offset/provider-byte oracle, final Redis and durable +cursor checks retained. No buffer increase, no production changes. + +Steady4000/s:55050/55430applied in34.276s, CPU34.239s; bounded replay backlog +exhausted after overflow. Steady5000/s:54939/55430applied in33.459s, CPU33.581s; +not certified. Captured low1000/high5000/low1000 burst33.253s plus2sclient loss +and10sslow reader:55430/55430ordered identities PASS, Redis/ACK10/10; caught up +7.126s after producer finish,4overflows/13148replayed then queue0. This is bounded +burst/drain evidence, not sustained5000 or production Kafka ingest capacity. +Per-reader saturation attribution remains limited; do not infer isolated queue +telemetry from aggregate overflow. Original provider clocks unchanged/test clock +explicit. Corrected profile harness self-time0.425s/10.90s; serial consumer CPU +remains the sustained bottleneck. Execution status remains NOT_FULL. + +TS serialization patch5970299 eliminates redundant legacy deep-copy;9unitPASS +and identical-operation microbenchmark measured38.44% less CPU for that function, +not a38% whole-system improvement. Harness evidence and commands indexed at +burst-recovery-20260927/adapter-capacity-checkpoint.json. Disposable clients removed; +no new image. Six anonymous-volume candidates retained without exact ownership, +not broadly deleted. Active/rollback images unchanged. No push/merge/release. diff --git a/rust/qdl-stream-gateway/examples/kn_trade_burst_gateway.rs b/rust/qdl-stream-gateway/examples/kn_trade_burst_gateway.rs index 958c2e2c..600a23f3 100644 --- a/rust/qdl-stream-gateway/examples/kn_trade_burst_gateway.rs +++ b/rust/qdl-stream-gateway/examples/kn_trade_burst_gateway.rs @@ -183,7 +183,7 @@ fn requirement(uid: &str, template: &Value) -> query::DataRequirement { recovery_policy: query::RecoveryPolicy::SnapshotAndReplay as i32, revision_policy: query::BarRevisionPolicy::Latest as i32, require_full_coverage: true, - require_final_bars: true, + require_final_bars: false, event_recency_policy: query::StalePolicy::Observe as i32, max_freshness_ms: template["requirement"]["max_freshness_ms"] .as_u64() @@ -583,7 +583,26 @@ async fn main() -> Result<(), Box> { #[cfg(test)] mod scheduling_tests { - use super::scheduled_seconds; + use super::{requirement, scheduled_seconds}; + + #[test] + fn trade_requirement_matches_production_facade_cursor_binding() { + let template = serde_json::json!({"requirement": { + "source_policy_id": "crypto_primary_v2", + "max_freshness_ms": 3000, + "max_session_liveness_ms": 45000 + }}); + let request = requirement("11111111-1111-5111-8111-111111111111", &template); + assert!(!request.require_final_bars); + assert!(request.require_full_coverage); + let actual = qdl_contracts::requirement::ValidatedRequirement::from_proto(&request) + .expect("valid production TRADE requirement"); + let mut mismatched = request.clone(); + mismatched.require_final_bars = true; + let old = qdl_contracts::requirement::ValidatedRequirement::from_proto(&mismatched) + .expect("old fixture remains syntactically valid"); + assert_ne!(actual.delivery.digest(), old.delivery.digest()); + } #[test] fn constant_and_burst_are_bounded_monotonic_schedules() { assert_eq!(scheduled_seconds(10000, 10000, 5000, 5000), 2.0); From 53beea1bb1a300bf5b565293b160015ecc194d77 Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Sun, 27 Sep 2026 19:37:14 +0000 Subject: [PATCH 10/17] Preserve RPC timeout at diagnostic observation boundary --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 39 +++++++++++++++++++++- scripts/probe_projector_stage_timing.py | 18 +++++++--- tests/test_probe_projector_stage_timing.py | 7 ++++ 3 files changed, 59 insertions(+), 5 deletions(-) diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index 511cc3fb..89d8ca5e 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -60478,7 +60478,7 @@ Status: IN_PROGRESS; EXECUTION_ACCEPTANCE_NOT_FULL remains unchanged. Read-only180s probe captured5543canonical records and3228Query reads on both replicas/real live+sandbox identities. BTC/OKX MARK had10rejections: all10 had a newer same-session/generation canonical pair observed before the request started. -Across the three inspected OKX products,40TRADE stale views also had newer +Across the three inspected Binance/OKX products,40TRADE stale views also had newer canonical data before request start. This proves downstream lag for those samples, not the exact stage responsible; no blanket quiet-market conclusion is valid. The raw topic returned ACL29; no alternative credential or ACL bypass attempted. @@ -60547,3 +60547,40 @@ not a38% whole-system improvement. Harness evidence and commands indexed at burst-recovery-20260927/adapter-capacity-checkpoint.json. Disposable clients removed; no new image. Six anonymous-volume candidates retained without exact ownership, not broadly deleted. Active/rollback images unchanged. No push/merge/release. + +Lineage reporting correction: the three traced UIDs include Binance BTC, +OKX BTC and OKX BNB, not three OKX products. UID/native venue mapping checked +against captured authenticated instrument templates. BTC/OKX10/10MARK finding +is unchanged;40TRADE lag findings span Binance and OKX. No evidence rewritten. + +Isolated native transport causal experiment: four arms verified6000/6000 captured +records, default fetch500/backoff1000 versus bounded20/10ms. Median transport +152.1->131.5ms; offset submission+final commit103.1->102.7ms; final commit alone +under1ms. It does not reproduce production361ms/744ms tail, omits Redis and RF3, +and does not justify a production tuning change. Evidence fetch-causal/summary.json; +opt-in instrumented scratch harness preserved outside Git with its orchestration. +Broker/client/network/executable removed. Disk used197753720832->197721161728bytes +includes concurrent host writes, not attributed entirely to cleanup. No images +built, no cap changes, active runtime images/start times unchanged. + +Targeted MARK10-product/two-replica fast matrix ran read-only with live TS identity. +All20reference cells have successful reads. One BinanceBTC reference was cancelled +at21.6ms by the six-second observer window, not a full service timeout; retain +this failed/censored receipt. Correct probe admission: observation end stops new +requests, admitted RPCs retain2.5s timeout; mark responses completing after window. +At most one RPC extends a window by2.5s. No quality/production policy changed. +First client attempt could not create evidence due UID permissions, before reads; +replacement client mounted only selected identity read-only and ran as container +root on read-only FS. Existing reference response times mostly8-10ms median. + +MARK fast matrix result:20/20product x replica cells returned successful execution +reference items. Initial209reference requests:208OK plus1observer-window cancel; +retain it. Targeted BinanceBTC follow-up after timeout-boundary fix:12/12OK, +6per replica, median8.95/9.80ms,max11.04/11.16ms. Other cells median7.85-10.00ms, +max21.42ms;11samples/cell, no per-cell p99. These are warm HTTP+SDK call durations +(the adjacent generic call establishes TLS), not TS Redis-commit latency. Generic +snapshot separately refused2BNB/OKX views; no gate or source timestamp changed. +Evidence mark-reference-matrix/summary.json and btc-followup.jsonl; original +receipts preserved. Observer helper unit6PASS. Both clients auto-removed, no new +image or production mutation. Actual TS runtime inspect: CPU/memory limits0 +(uncapped);2->4CPU budget increase applies only to isolated worker tests. diff --git a/scripts/probe_projector_stage_timing.py b/scripts/probe_projector_stage_timing.py index bcd1df0e..5f7bc450 100644 --- a/scripts/probe_projector_stage_timing.py +++ b/scripts/probe_projector_stage_timing.py @@ -12,6 +12,12 @@ from scripts.probe_exact_market_lineage import Evidence +def admitted_query_timeout(remaining_seconds): + # Stop admitting at the observation boundary, but do not shorten an RPC's + # normal timeout and misclassify window cancellation as service failure. + return 2.5 if remaining_seconds > 0 else None + + def reference_metadata(response): results = [] for item in response.get("results", []): @@ -86,11 +92,14 @@ async def reference_batch(self, requirements, *, consumer_id, require_all): for replica, transport, client in clients: for path in ("generic_snapshot", "ts_reference_batch"): remaining = deadline - time.monotonic() - if remaining <= 0: + request_timeout = admitted_query_timeout(remaining) + if request_timeout is None: return - row = dict(pair_id=pair_id, replica=replica, path=path, request_started_at_ns=time.time_ns()) + row = dict(pair_id=pair_id, replica=replica, path=path, + request_started_at_ns=time.time_ns(), request_timeout_seconds=request_timeout, + observation_remaining_seconds=remaining) try: - async with asyncio.timeout(min(remaining, 2.5)): + async with asyncio.timeout(request_timeout): if path == "generic_snapshot": await client.snapshot(req) else: @@ -98,7 +107,8 @@ async def reference_batch(self, requirements, *, consumer_id, require_all): row["status"] = "TYPED_RESPONSE" except Exception as error: row.update(status="REFUSED", code=getattr(error, "code", type(error).__name__), diagnostics=getattr(error, "diagnostics", None)) - evidence.put("query", **row, response_at_ns=time.time_ns(), exact_view=transport.last) + evidence.put("query", **row, response_at_ns=time.time_ns(), exact_view=transport.last, + completed_after_observation_window=time.monotonic() > deadline) await asyncio.sleep(.5) finally: for _, _, client in clients: diff --git a/tests/test_probe_projector_stage_timing.py b/tests/test_probe_projector_stage_timing.py index e601d078..9b85abae 100644 --- a/tests/test_probe_projector_stage_timing.py +++ b/tests/test_probe_projector_stage_timing.py @@ -5,6 +5,13 @@ class StageTimingTests(unittest.TestCase): + def test_window_end_does_not_shorten_admitted_rpc_timeout(self): + from scripts.probe_projector_stage_timing import admitted_query_timeout + self.assertEqual(admitted_query_timeout(0.021), 2.5) + self.assertEqual(admitted_query_timeout(180), 2.5) + self.assertIsNone(admitted_query_timeout(0)) + self.assertIsNone(admitted_query_timeout(-1)) + def test_reference_preserves_problem_components_without_price_or_secrets(self): import json response = {"partial": True, "results": [{"status": "ERROR", "problem": {"code": "DATA_STALE"}}, From fb58ae67adcbae2f233fe92fc2ea58f6b599b04f Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Sun, 27 Sep 2026 19:52:46 +0000 Subject: [PATCH 11/17] docs: record exact consumer burst limits and unfinished recovery --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 27 +++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index 89d8ca5e..d33526f1 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -60584,3 +60584,30 @@ Evidence mark-reference-matrix/summary.json and btc-followup.jsonl; original receipts preserved. Observer helper unit6PASS. Both clients auto-removed, no new image or production mutation. Actual TS runtime inspect: CPU/memory limits0 (uncapped);2->4CPU budget increase applies only to isolated worker tests. + +#### Final capacity checkpoint - execution acceptance remains NOT_FULL + +TS source-only checkpoint fa9a597 adds opt-in fixed product-family workers inside +one market_data_service, no new service topology. Default1 unchanged; worker crash +resume deliberately refuses committed cursors until atomic whole-family state/ +watermark and nonregressing replay are implemented. This is unfinished scope, +not an approval blocker and not production-ready. Combined309unitPASS4.42s. + +Four-worker fresh-start replay of authentic capture:4000/s and5000/s both applied +55430/55430 ordered identities with Redis and durableACK parity. Worst route p99 +scheduled-offer-to-Redis1220.793ms at4k and4068.205ms at5k; finalACK drain461.032ms +and3240.890ms respectively.5k had2RATE_LIMITED recoveries. These short receipts do +not certify sustained60s capacity, crash recovery, or5000/sKafka ingress/fanout. +Evidence workers-4000-fresh-r10 and workers-5000-fresh-r10 under existing burst root. +Profile worker0 completed; pydantic conversion, SDK validation, async scheduling +and Redis serialization remain distributed costs. No notification coalescing, +buffer expansion, freshness relaxation or production rollout used to forcePASS. + +All disposable clients/Redis/gateway containers removed; Redis testdata tmpfs, +no new images built. Active/rollback retained, no broadprune. Publishedv2.2.0 +main8299838 unchanged. CanonicalDL fix/live-consumer-burst-recovery and canonicalTS +fix/data-layer-r10-consumer-handoff; owner source edits preserved. No newworktree, +push/merge/tag. RuntimeQuery0ed57643/Stream65918a65/TS1d86d3e8 unchanged; consumer +runtime remains paperidentity, additive live/sandbox readidentities are not an +execution-capacity certificate. Next required work remains monotonic atomic resume, +then sustained4k/5k and bounded actualconsumer acceptance, not another phase. From baa50b5711ca57d8b036931a08f5933c6c7eaa80 Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Mon, 28 Sep 2026 02:38:13 +0000 Subject: [PATCH 12/17] docs: record consumer atomic recovery evidence and rollout boundary --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 46 +++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index d33526f1..54444006 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -60611,3 +60611,49 @@ push/merge/tag. RuntimeQuery0ed57643/Stream65918a65/TS1d86d3e8 unchanged; consum runtime remains paperidentity, additive live/sandbox readidentities are not an execution-capacity certificate. Next required work remains monotonic atomic resume, then sustained4k/5k and bounded actualconsumer acceptance, not another phase. + +2026-09-28 approved narrow consumer runtime packet, READY_TO_APPLY. +TS4e04c96 image sha256:08e0c9072f7c72d4c2ee5752018d25eff421be5166b0b5a931636528028487a8; +base/rollback sha256:1d86d3e820795a50b2539b09a6d35b102efd461e7bd7dec1801730a5daffdc46. +Only trading_system/market_data (container market_data_service). Preserve networks, +allcurrentenv except MARKET_DATA_FIXED_WORKERS=4, mounts,TLS,identitypaperrev10; +noother TS/DataLayer/alpha/order/Kafka/Redisflush/SQLite mutation. Stop oldconsumer, +backup andretainlegacycursor; copy preflightedownership into separatefixed-workers-v1 +cursor namespace. FreshQuerybootstrap validatesexistingexecutionwatermark, never +blindresume legacycursor. V2routes60,workerallocation24/12/12/12,82VNlegacyretained. +Candidatepackaging342PASS46.04s (actualinstalledSDK,notSDKsourcemount). +Packet/evidence: burst-recovery-20260927/atomic-rollout-4e04c96. +Currentbaseline59/60 withOKXBNB BOOK_SNAPSHOT DATA_STALE; BOOK_DELTA last-event +age~2494s/sessionLIVE/eligiblefalse. Separatefindingretained,notTRADElatencyclaim. +After rollout observe300s health/cache/logs/cursor/workertelemetry; failclosed. +Rollback recreateonlymarket_data usingrollback.compose.json,legacycursor remains. +PublishedDLv2.2.0 unchanged, noFulluntilconsumeracceptance. + +2026-09-28 closure checkpoint: IMPLEMENTED / ISOLATED_TESTED, RUNTIME_PENDING, +EXECUTION_ACCEPTANCE_NOT_FULL (no production-certification claim). +TS source4e04c96; image08e0c9072f7c72d4c2ee5752018d25eff421be5166b0b5a931636528028487a8. +Final TLS5k aggregate10TRADEroutes304865exact, maxroutep99678.120ms, finalACK +354.252ms afteroffer; client2.475avgcores/362.64MiBpeak/noCPUthrottle. +4k243892exact,p99431.449ms,ACK222.782ms; client1.991cores/341.61MiB. +Finalsame-source burst1000/5000/1000 plus2sdisconnect/10sslowreader:110860exact, +1reconnect,allRedis+ACKhashesmatch,p991950.866ms,finalACK37.279ms; zeroorders. +These are scheduledfixtureoffer-to-actualconsumerRedis latencies, notprovider +eventage/liveeligibility,not5000perconsumerorfullKafkaingest/fanoutcertification. +Index: ~/.local/state/qdl-v2/burst-recovery-20260927/atomic-rollout-4e04c96/acceptance-index.json. +Production stop was REJECTED by automatic approval review beforeexecution: exact +market_data_service outage/recreate impact needs explicit confirmation. Async +question sent withdigest,4workers,cursorbootstrap,briefoutage and1d86d3e rollback. +Noindirectworkaround. ConfirmedoldserviceRUNNING,1d86d3e,restart0; no300sactual +consumeracceptance orliveidentityactivation performed. Existing BNB/OKXsnapshot +stale remainsseparatebaselinefinding; noFullreleaseclaim. +Cleanup: allcapture gateway/client/Rediscontainers auto-removed; exact disposable +ts-atomic-recovery-redis andts-atomic-recovery-20260928 removed. Zero ts-atomic/ +qdl-atomic containers remain. Removed21testprivatekeys+238662Bbuildcontext; +retainedcandidate08e0c907 andactive/rollback1d86d3e forpendingpacket, publicTLS +certs/receipts/captures. No broadprune,sharedvolume/networkcleanup orworktree +delete. BuildKitcleanup deferred untilruntimeoutcome toavoidsharedcache removal. +Hostusedbytes before202716180480,after203296567296 (backgroundgrowth; doNOTclaim +netreclaimedspace). DLruntimeQuery0ed57643/Stream65918a65 unchanged; published +v2.2.0 main8299838. CanonicalDLbranchfix/live-consumer-burst-recovery fb58ae6; +canonicalTSbranchfix/data-layer-r10-consumer-handoff4e04c96; otheractiveTSfeature +worktrees untouched. No push/merge/release. Ownercompose/symbols/planhunks retained. From a1a9b41332413b5dd74566ebe042ef2b8d08fec8 Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Mon, 28 Sep 2026 03:56:15 +0000 Subject: [PATCH 13/17] docs: record BNB L2 freshness evidence and consumer handoff result --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 127 ++++++++++++++++++++++ 1 file changed, 127 insertions(+) diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index 54444006..53ca135b 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -60657,3 +60657,130 @@ netreclaimedspace). DLruntimeQuery0ed57643/Stream65918a65 unchanged; published v2.2.0 main8299838. CanonicalDLbranchfix/live-consumer-burst-recovery fb58ae6; canonicalTSbranchfix/data-layer-r10-consumer-handoff4e04c96; otheractiveTSfeature worktrees untouched. No push/merge/release. Ownercompose/symbols/planhunks retained. + +2026-09-28 read-only BNB/OKX L2 investigation (sidecar, no source/runtime change). +Scope only BNB-USDT-SWAP UIDf2e37e2b-1386-5a32-9b79-0fd39ec7a5a3; +actual TS paperidentity/manifestrev10, existingSDK RestQueryTransport inside +market_data_service image1d86d3e at probe time, credential/TLS read in-process, +no secret output/cursor writes/stream/order/C2. Query0ed57643,Stream65918 unchanged. +TwoQueryreplicas, status+snapshot forBOOK_SNAPSHOT andBOOK_DELTA (8reads): +Snapshot bothSTALE,eventSTALE,sessionNOT_APPLICABLE,complete=true,gap=false, +eligible=false,LAST_EVENT_STALE. Offset274362767; source_event1790560208308000000, +received1790560208558479471. At request1790566026583565907 freshness5818360ms; +Query2 subsequentfreshness5818413ms. Both snapshot reads correctly DATA_STALE. +Delta bothLIVE/eventSTALE/sessionLIVE,eligible=false,complete=true,gap=false; +watermark274362869,source_event1790560208908000000,received1790560208957310113, +generation66791,sequence_verified=true,native_sequence_end17411513614, +snapshot_sequence17402419892. Sessionage7..999ms; eventage5817779..5817831ms. +Both exactsameview contractcatalog11/config19/authority1; authoritative=true, +sourceidokx-swap-bnb-usdt-swap-book-primary-v2. Snapshotcalls8.29/10.84ms, +delta8.96/8.77ms; statusfirstcold88.55ms/others8.29..14.13ms. Tinybounded +sample only, notp99/acceptance. Age97min is real olddata, notrequestlatency. +OneapprovedpublicOKX read viaexistingOkxRestClient.get('/api/v5/market/books', +instIdBNB-USDT-SWAP,sz100,bucketmarket,attempts1,timeout3s) fromQuery1: +start1790566065461403848/end1790566065585639096;124.235mscall; +providerts1790566065552,33.639msageatreceive,seqId17412847137,100levels/side, +bid769.2/ask769.3. This positively disproves providerquiet interpretation for +current stale book. Strictsnapshot rejection is correct, mustnotwiden60s gate +or relabel stale bookeligible fromsessionheartbeat. +Source review: Rust materializedsnapshot generated only onverifiedDeltaApplied/ +Keepalive via due_l2_materialized_snapshot; core canrequestresync; OKX ingestor +book_session_bootstrap usesInitialSnapshotAndGapResync (notperiodicRESTanchor). +ConfiguredBNBbooks exists,depth100,snapshot_refresh_seconds30. This alone does +NOT prove whichpipeline stagefroze. Boundedcorelogs show aggregate growing +REJECTED_AWAITING_SNAPSHOT but noBNBspecificstatechange inselected120m logs; +oneidentifiedsequencegap belongedtoBINANCEBTCquarterly, NOTBNB. DoNOTattribute +aggregatecountertoBNB. No raw/canonical-topic capture inthisboundedinvestigation: +latestQuerymaterialization is old, but root betweenproviderWSingest/core/resync/ +projector remains unlocalized. Nextnarrowstep: traceonlyBNBraw/canonical/head +andresyncownership; donotchangeSDKsnapshotsemantics orconsumerworkers tohideit. +Investigationdone, BNB executionreadinessUNRESOLVED, notFULL. No testresources +created; cleanupnotneeded, active/rollback/artifactsunchanged bysidecar. Main +separately reportsTSrollout693d866 duringinvestigation; probeabove predatesit. + +Sidecar runtime provenance correction: final dockerinspect still observed actual +market_data_service sha256:1d86d3e820795a50b2539b09a6d35b102efd461e7bd7dec1801730a5daffdc46 +running/restart0. Main message mentioned693d866 candidate; sidecar didnotverify +thatimageactive and doesnotassertsuccessfulrollout. No mutation bysidecar. + +2026-09-28 bounded read-only BNB localization follow-up, DONE_WITH_UNLOCALIZED_LAYER. +Within5min scope: inspected exactBNBbooks runtime configs acrosscore.json/core-002/ +core-003, allhaveUIDf2e37e2b/nativebooks/depth100/materializedsnapshot1000ms, +providerrefresh30s; ingestorbindingpresent. Bounded130m/tail20000percore logs +contain noBNBstatechange. Observedsequencegap isBinanceBTCquarterly, explicitly +NOTBNB evidence. Ingestor running/restart0, boundedstdout logs empty, json-file +rotation3x50MB; absenceoflog isnotabsenceofrawframes. Projector1/2 boundedstatus +haserrors{}, stageAinputs=outputs, stageBlatestappliedincreasing; theseaggregate +numbers donotruleoutproduct-specificfailure. No raw/canonical BNBrecord captured +inthiswindow, no existing authorized reader wasestablished for exacttopictrace; +no credentialescalation/ACLchange/groupjoin/offsetcommit performed. +Source review: core due_l2_materialized_snapshot publishes onlyverifiedDeltaApplied/ +Keepalive. OKXingestorInitialSnapshotAndGapResync disablesperiodicRESTanchor; +core request_resync islocaladapterstate. This suggestscheckingfeedbackfromcore +resynctoOKXsubscription, but doesNOTproveBNBcurrentlyresyncing. DoNOTdeclare +corethecause fromaggregateREJECTED_AWAITING_SNAPSHOT. Correct narrowbranching: +freshBNBraw+noBNBcanonical ->core/continuity/resync; freshBNBcanonical+oldlatest -> +projector/materialization; noBNBraw despitefreshprovider ->ingestorsubscription. +Exactmissing evidence: boundedBNBrawseq/time/session+canonicalseq/offset+latest +onmatchinginterval, thenownerforrequiredresync. No source/runtimechanges, no +furtherproviderrequests (previoussinglepublicread alreadyprovesfreshvenuebook), +noresourcescreated/cleanupneeded, noFullclaim. 60routeclosure stillblockedby +BNBbookmaterialization; doNOTwidenfreshness orrestartunrelatedservices. + + +### 2026-09-28 Final consumer-only rollout and 300-second receipt + +Status: `CONSUMER_RECOVERY_PASS_OBSERVED_300S`; aggregate +`EXECUTION_ACCEPTANCE_NOT_FULL`. No new release, push or merge. + +Approved scope completed: only market_data_service replaced at 03:45:20 UTC +with TS source 0a5400a, image sha256:1037c56a48d65156bb64136db78325678e4940b51bc12dbee050162cf9549217. +Four fixed workers; existing config/mounts and paper identity revision 10 kept. +Rollback image 1d86d3e and legacy cursor backup retained. No live-realm +activation, alpha/order action, Data Layer rollout, reset or flush performed. +Two earlier candidate attempts were rolled back, not counted as successful: +08e0c907 exposed stale Query refresh versus newer stream watermark; 693d866 +exposed QUOTE replay versus TRADE/BOOK-only health handling. Source corrections +10701df and 0a5400a retain strict source/generation/freshness fencing, make +verified older refresh a no-op, and keep replay health separate from execution +eligibility. Actual final-image affected suite: 197 passed in 10.75 seconds. + +Final observer exited 0 after 302 seconds; data window 300.009 seconds, +151 samples / 11 distinct service heartbeats. All heartbeat observations: +59/60 session-ready, four worker telemetry epochs present, no fallback. +Execution-ready counter was 48-49, NOT 59-60: session readiness must not be +reported as executable-price availability. No observed watermark regression or +execution checksum violation; cursor errors 0, ACK delta 26,897 and commit +delta 13,749. These sampled invariants do not certify every historical write. +CPU average 0.720 core, peak cgroup memory 434.24 MiB, throttle delta 0; +runtime running, restart 0, OOM false. Local Redis MGET p50 1.403 ms/p99 5.350 ms +is NOT Data Layer endpoint latency or event-to-Redis commit latency. No new +forced disconnect or stress run in this window; inherit separately documented +4k/5k captured-event stress and disconnect/recovery evidence, not live freshness. + +Remaining true defect: OKX BNB BOOK_SNAPSHOT repeatedly DATA_STALE; both Query +replicas serve matching book roughly 97 minutes old at diagnostic time, while +one provider read returned a 33.639 ms-old book. BOOK_DELTA session LIVE also +does not make that old book executable. Provider quiet is disproved. Exact +fault layer between raw ingest, core/resync and materialization is not yet +localized. Do not widen freshness, restart blindly or declare overall FULL. +Next narrow task is matching BNB raw/canonical/cache evidence and repairing +that specific path; no reason to repeat all previously certified products. +Candidate consumer remains active because this pre-existing book rejection +persists correctly, while the consumer recovery integration fixes passed. + +Cleanup completed within this task: disposable Redis/client containers and +network removed, two superseded candidate images 08e0c907/693d866 removed, +owned build contexts removed, exact two owned BuildKit IDs pruned (117 kB and +44.77 kB). Retain active 1037c56a plus explicit rollback 1d86d3e. No broad prune, +volume deletion, worktree removal or unrelated service change. Filesystem used +205,413,998,592 bytes before cleanup; 204,792,332,288 after (concurrent filesystem +activity means the whole difference is not attributable to this cleanup). + +Canonical TS /home/bobby/trading_system on fix/data-layer-r10-consumer-handoff; +canonical DL /home/bobby/data_layer on fix/live-consumer-burst-recovery. Existing +other-feature TS worktrees remain unchanged. Published DL v2.2.0/main8299838 +unchanged; Query 0ed57643 and Stream 65918a65 unchanged. Owner compose/symbols +and unrelated plan edits preserved. Final receipt outside Git: +`/home/bobby/.local/state/qdl-v2/burst-recovery-20260927/atomic-rollout-4e04c96/runtime-0a5400a/acceptance-receipt.json` +SHA256: 60b99f03dc80179ed39ed28551bc3be44df5479dc95232728c5b807c02efc028. From 166cda1195f9f4ad3e8647150d3ebd594a01d95d Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Mon, 28 Sep 2026 04:22:14 +0000 Subject: [PATCH 14/17] fix: preserve verified L2 continuity across processing backlog --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 55 +++++ rust/qdl-realtime-core/src/lib.rs | 235 ++++++++++++++++------ 2 files changed, 233 insertions(+), 57 deletions(-) diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index 53ca135b..a6c04387 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -60784,3 +60784,58 @@ unchanged; Query 0ed57643 and Stream 65918a65 unchanged. Owner compose/symbols and unrelated plan edits preserved. Final receipt outside Git: `/home/bobby/.local/state/qdl-v2/burst-recovery-20260927/atomic-rollout-4e04c96/runtime-0a5400a/acceptance-receipt.json` SHA256: 60b99f03dc80179ed39ed28551bc3be44df5479dc95232728c5b807c02efc028. + + +### 2026-09-28 BNB OKX L2 recovery correction + +Status: IN_PROGRESS. Owner requests narrow diagnosis and repair, not catalogue retest. +Guide: Kafka-native architecture review K4-T05 (snapshot/delta/resync) and K5 +runtime safety. Trace bounded raw -> canonical -> cache metadata first; fix +the owning shared Rust boundary without changing checksum/sequence/generation +or freshness. Test reproduced failure, recovery/duplicate/gap and symbol +isolation, then two Query replicas and TS consumer. No forged freshness, +offset reset, Redis flush, unrelated rollout, alpha or order activation. +Record role/digest/config and rollback before any required deployment. Retain +active image as rollback; clean task resources only. FULL requires verified +fresh book, not session heartbeat. apply_patch helper is broken; Rule32 exact +append fallback used, owner changes preserved. + + +BNB localization and approved runtime packet checkpoint: +- Core identity, manual Kafka assign, no group join/commit: live raw12s =125BNB + books of9898records, generation66791, partition4. Historical9s around freeze + contains90 contiguous BNBframes. Last accepted17411513614 -> nextraw17411513655 + withprev17411513614. Canonical12s =7018records, zeroBNBbook but221otherBNB. +- Core1 processing age rose20.6s ->53.7s at01:50UTC; quarantine3->343 then + awaiting-snapshot1697 persisted after backlog returned below1s. Source guard + resyncs on30s processing age, before applying otherwise contiguousdelta. + Exact BNB quarantine attribution not captured (readACL denied; no bypass). +- Correctness repair: lossless processing with original timestamps; local queue + age is not provider continuity failure. Real sequence/gap/generation rejection + stays. Reader freshness still rejects old records. No heartbeat-as-price. +- Owner approved exact rollout3cores, then onlyOKXingestor reconnect. Keepall + config/TLS/offsets/cache and servicequotas unchanged. Rollbackthreecores to + sha256:7fe348060734e4f51824b02faed7020465bb8dc754ad5299cb88befba7f9f69f. + OKXingestor stayssameimage; reconnect getsrealWSsnapshot afterlastrebalance. + This briefly affectsallOKXfeedsonthatservice, notonlyBNB. NoQuery/Stream/TS, + Kafkaadmin, identity, Redis, V1 ororder mutations. Candidate digest follows + completedtests/build. Existingcompose at releases/v2.2.0-02cd827/ + producer-candidate.compose.json; exactcurrentruntime inspected beforeapply. + + +Source test checkpoint (before deployment): lossless L2 replay patch plus four +new dual-venue regressions in rust/qdl-realtime-core/src/lib.rs. Old guard restored +in an outside-Git overlay makes the new contiguous replay regression FAIL; +correct source passes. Cargo cache initially retained the overlay single-package +binary; package-scoped offline clean then broad rebuild removed ambiguity. +Final affected Rust lib/bin gates:145passed,0failed,1ignored (provider admission +Redis integration, unchanged). Clippy realtime-core all-targets -Dwarnings PASS. +Reader/cache/SDK with disposable Redis:40passed,0skip (initial run without Redis +was11pass/29skip and is not counted as complete evidence). +An optional run of the unchanged ignored provider-admission test FAILED at +provider_admission.rs:706: it uses50,000ns cooldown rounded to1ms Redis TTL; +second network round trip can expire it. No admission code/policy changed and +no claim of full admission recertification. Report separately, do not widen +production cooldown or hide the test. Disposable Redis will be removed. +No runtime mutation yet. Source patch does not make historical prices executable; +Query/SDK still evaluate original event age and source/session fences. diff --git a/rust/qdl-realtime-core/src/lib.rs b/rust/qdl-realtime-core/src/lib.rs index 176e31da..fe5cc214 100644 --- a/rust/qdl-realtime-core/src/lib.rs +++ b/rust/qdl-realtime-core/src/lib.rs @@ -70,12 +70,6 @@ impl L2Binding { .unwrap_or_else(|| self.snapshot_refresh_seconds.saturating_mul(1_000)) } - fn raw_receipt_is_expired(&self, received_at_ns: i64, processing_at_ns: i64) -> bool { - let max_age_ns = self.snapshot_refresh_seconds.saturating_mul(1_000_000_000); - let max_age_ns = i64::try_from(max_age_ns).unwrap_or(i64::MAX); - processing_at_ns.saturating_sub(received_at_ns) > max_age_ns - } - fn validate(&self, binding: &CoreBinding) -> Result<(), CoreError> { let materialized_snapshot_interval_ms = self.materialized_snapshot_interval_ms(); if !(1..=10_000).contains(&self.depth_per_side) @@ -1020,24 +1014,17 @@ impl RealtimeCore { binding: &CoreBinding, raw: RawProviderEnvelope, payload: Value, - processing_at_ns: i64, + _processing_at_ns: i64, transport_offset: Option, ) -> ProcessBatch { let materialized_at_ns = raw.received_at_ns; let Some(l2) = binding.l2.as_ref() else { unreachable!("process_l2 only accepts an L2 binding") }; - if l2.raw_receipt_is_expired(raw.received_at_ns, processing_at_ns) { - if let Some(adapter) = self.l2_adapters.get_mut(binding_key) { - adapter.request_resync(raw.connection_generation); - } - return self.quarantine( - &raw, - QuarantineReason::SemanticInvalid, - "L2 raw receipt exceeded provider renewal bound; resync required", - processing_at_ns, - ); - } + // Processing backlog is not a provider continuity failure. Replay every + // validated book frame losslessly with its original source/receipt times; + // readers still enforce freshness. Dropping an aged delta and clearing + // the book strands a healthy OKX socket waiting for a new snapshot. let transport = match TransportProtocol::try_from(raw.transport_protocol) { Ok(value) => value, Err(_) => { @@ -2581,48 +2568,182 @@ mod tests { assert_eq!(evidence.reason, QuarantineReason::SequenceGap as i32); } - #[test] - fn stale_l2_raw_is_quarantined_without_redating_and_fresh_snapshot_recovers() { - let binding = binance_book_binding(); - let mut core = core(binding.clone(), true); - let stale = with_transport( - raw( - &binding, - br#"{"lastUpdateId":100,"bids":[["60000","1"]],"asks":[["60001","1"]]}"#, - 1, - ), - TransportProtocol::Http, + // Synthetic protocol fixtures: processing time is deliberately independent + // from provider and receipt time, as during Kafka backlog recovery. + fn delayed_book_frame( + binding: &CoreBinding, + snapshot: bool, + sequence: u64, + previous: u64, + generation: u64, + received_at_ns: i64, + bid_quantity: &str, + ) -> RawProviderEnvelope { + let time_ms = received_at_ns / 1_000_000; + let payload = if binding.venue == "OKX" { + serde_json::json!({ + "arg": {"channel": "books", "instId": binding.native_symbol}, + "action": if snapshot { "snapshot" } else { "update" }, + "data": [{"seqId": sequence.to_string(), + "prevSeqId": if snapshot { "-1".into() } else { previous.to_string() }, + "ts": time_ms.to_string(), + "bids": [["60000", bid_quantity, "0", "1"]], + "asks": [["60001", "1", "0", "1"]]}] + }) + } else if snapshot { + serde_json::json!({"lastUpdateId": sequence, "E": time_ms, + "bids": [["60000", bid_quantity]], "asks": [["60001", "1"]]}) + } else { + serde_json::json!({"s": binding.native_symbol, "U": previous, + "u": sequence, "pu": previous, "E": time_ms, + "b": [["60000", bid_quantity]], "a": [["60001", "1"]]}) + }; + let frame = raw_with_receipt( + binding, &serde_json::to_vec(&payload).unwrap(), generation, received_at_ns, ); - let processing_at_ns = stale.received_at_ns + 30_000_000_001; - let rejected = core.process(stale, processing_at_ns).unwrap(); - assert!(rejected.canonical.is_empty()); - assert_eq!(rejected.quarantines.len(), 1); - let evidence = - QuarantineRecord::decode(rejected.quarantines[0].payload.as_slice()).unwrap(); - assert_eq!(evidence.reason, QuarantineReason::SemanticInvalid as i32); - assert_eq!(evidence.quarantined_at_ns, processing_at_ns); - assert!(evidence.safe_summary.contains("renewal bound")); + if snapshot && binding.venue == "BINANCE" { + with_transport(frame, TransportProtocol::Http) + } else { + frame + } + } - let mut fresh_delta = raw( - &binding, - br#"{"s":"BTCUSDT","U":99,"u":101,"pu":98,"E":1001,"b":[["60000","2"]],"a":[["60001","1"]]}"#, - 2, - ); - fresh_delta.received_at_ns = processing_at_ns; - let buffered = core.process(fresh_delta, processing_at_ns).unwrap(); - assert!(buffered.canonical.is_empty()); + #[test] + fn delayed_l2_contiguous_replay_preserves_times_state_and_lossless_publications() { + for binding in [okx_book_binding(), binance_book_binding()] { + let mut delayed = core(binding.clone(), true); + let mut timely = core(binding.clone(), true); + let receipt = 1_786_352_400_000_000_000; + let now = receipt + 60_000_000_000; + let frames = [ + delayed_book_frame(&binding, true, 100, 0, 7, receipt, "1"), + delayed_book_frame(&binding, false, 101, 100, 7, receipt + 1_000_000, "2"), + delayed_book_frame(&binding, false, 102, 101, 7, receipt + 2_000_000, "0"), + delayed_book_frame(&binding, false, 103, 102, 7, now, "3"), + ]; + for (offset, frame) in frames.into_iter().enumerate() { + let original_receipt = frame.received_at_ns; + let expected = timely.process_at_transport_offset( + frame.clone(), original_receipt, offset as u64, + ).unwrap(); + let actual = delayed.process_at_transport_offset( + frame.clone(), now, offset as u64, + ).unwrap(); + assert!(actual.quarantines.is_empty(), "{}", binding.venue); + assert_eq!(actual.canonical.len(), expected.canonical.len()); + for (record, expected) in actual.canonical.iter().zip(&expected.canonical) { + assert_eq!(record.payload, expected.payload); + assert_eq!(record.accepted_at_ns, original_receipt); + let event = EventEnvelope::decode(record.payload.as_slice()).unwrap(); + assert_eq!(event.received_at_ns, original_receipt); + assert_eq!(event.normalized_at_ns, original_receipt); + assert_eq!(event.published_at_ns, original_receipt); + assert_eq!(event.source_event_time_ns, original_receipt); + } + if offset == 1 || offset == 2 { + assert!(!actual.canonical.is_empty(), "delayed deltas cannot be skipped"); + } + if offset == 2 { + assert!(delayed.l2_adapters[&binding.key()].core().view().unwrap().bids.is_empty()); + let duplicate = delayed.process_at_transport_offset(frame, now, offset as u64).unwrap(); + assert!(duplicate.canonical.is_empty()); + assert_eq!(duplicate.duplicates, 1); + } + } + let view = delayed.l2_adapters[&binding.key()].core().view().unwrap(); + assert_eq!(view.last_sequence, 103); + assert_eq!(view.snapshot_sequence, 100); + assert_eq!(view.bids.len(), 1); + } + } - let mut fresh_snapshot = with_transport( - raw( - &binding, - br#"{"lastUpdateId":100,"bids":[["60000","2"]],"asks":[["60001","1"]]}"#, - 2, - ), - TransportProtocol::Http, - ); - fresh_snapshot.received_at_ns = processing_at_ns; - let recovered = core.process(fresh_snapshot, processing_at_ns).unwrap(); - assert_eq!(recovered.canonical.len(), 1); + #[test] + fn delayed_l2_real_gap_still_requires_provider_snapshot() { + for binding in [okx_book_binding(), binance_book_binding()] { + let mut core = core(binding.clone(), true); + let receipt = 1_786_352_400_000_000_000; + let now = receipt + 60_000_000_000; + for frame in [ + delayed_book_frame(&binding, true, 100, 0, 7, receipt, "1"), + delayed_book_frame(&binding, false, 101, 100, 7, receipt, "2"), + ] { + assert!(core.process(frame, now).unwrap().quarantines.is_empty()); + } + let gap = core.process(delayed_book_frame( + &binding, false, 110, 109, 7, receipt, "3", + ), now).unwrap(); + assert!(gap.canonical.is_empty()); + assert_eq!(gap.quarantines.len(), 1); + assert!(core.l2_adapters[&binding.key()].core().view().is_none()); + let after = core.process(delayed_book_frame( + &binding, false, 111, 110, 7, now, "4", + ), now).unwrap(); + assert!(after.canonical.is_empty()); + for frame in [ + delayed_book_frame(&binding, true, 200, 0, 8, now, "5"), + delayed_book_frame(&binding, false, 201, 200, 8, now, "6"), + ] { + assert!(core.process(frame, now).unwrap().quarantines.is_empty()); + } + assert_eq!(core.l2_adapters[&binding.key()].core().view().unwrap().last_sequence, 201); + } + } + + #[test] + fn delayed_l2_invalid_level_still_invalidates_verified_state() { + for binding in [okx_book_binding(), binance_book_binding()] { + let mut core = core(binding.clone(), true); + let receipt = 1_786_352_400_000_000_000; + let now = receipt + 60_000_000_000; + for frame in [ + delayed_book_frame(&binding, true, 100, 0, 7, receipt, "1"), + delayed_book_frame(&binding, false, 101, 100, 7, receipt, "2"), + ] { + core.process(frame, now).unwrap(); + } + let invalid = core.process(delayed_book_frame( + &binding, false, 102, 101, 7, receipt, "-1", + ), now).unwrap(); + assert!(invalid.canonical.is_empty()); + assert_eq!(invalid.quarantines.len(), 1); + assert!(core.l2_adapters[&binding.key()].core().view().is_none()); + let fresh = core.process(delayed_book_frame( + &binding, false, 103, 102, 7, now, "3", + ), now).unwrap(); + assert!(fresh.canonical.is_empty()); + } + } + + #[test] + fn delayed_l2_generation_change_requires_anchor_and_rejects_old_generation() { + for binding in [okx_book_binding(), binance_book_binding()] { + let mut core = core(binding.clone(), true); + let receipt = 1_786_352_400_000_000_000; + let now = receipt + 60_000_000_000; + for frame in [ + delayed_book_frame(&binding, true, 100, 0, 7, receipt, "1"), + delayed_book_frame(&binding, false, 101, 100, 7, receipt, "2"), + ] { + core.process(frame, now).unwrap(); + } + let unanchored = core.process(delayed_book_frame( + &binding, false, 201, 200, 8, receipt, "3", + ), now).unwrap(); + assert!(unanchored.canonical.is_empty()); + assert!(core.l2_adapters[&binding.key()].core().view().is_none()); + let anchor = core.process(delayed_book_frame( + &binding, true, 200, 0, 8, receipt, "4", + ), now).unwrap(); + assert!(!anchor.canonical.is_empty()); + let old = core.process(delayed_book_frame( + &binding, true, 300, 0, 7, now, "99", + ), now).unwrap(); + assert!(old.canonical.is_empty()); + assert_eq!(old.filtered_outcome, Some("IGNORED_STALE_GENERATION")); + let view = core.l2_adapters[&binding.key()].core().view().unwrap(); + assert_eq!(view.generation, 8); + assert!(view.last_sequence == 200 || view.last_sequence == 201); + } } #[test] From 461577b6ead5a1383c9983ab924d6a36bdce08ab Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Mon, 28 Sep 2026 04:36:10 +0000 Subject: [PATCH 15/17] docs: record BNB L2 rollout recovery and exact eligibility limits --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 65 +++++++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index a6c04387..f82dae25 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -60839,3 +60839,68 @@ no claim of full admission recertification. Report separately, do not widen production cooldown or hide the test. Disposable Redis will be removed. No runtime mutation yet. Source patch does not make historical prices executable; Query/SDK still evaluate original event age and source/session fences. + +Runtime packet ready: source166cda1; immutable core image sha256:f2040ac9e9380d067e8bcb4c965b1d7ea39c181c08993b7dba1b096337f7ca79 +Verified packaged binary SHA256 85b56bbb49fb46aac163744246993cd4f9504d807efd7ab02856e22fbfa125e5. +Roll three cores sequentially using bnb-l2-recovery-20260928/candidate.compose.json, +then same-image OKX ingestor once after rebalance; rollback.compose.json has +verified current image/environment/command/CPU/RAM for all four roles. +All other services untouched. Owner explicitly approved this exact packet. + + +### 2026-09-28 BNB L2 repair deployed; scoped acceptance result + +Status: `BNB_L2_FREEZE_RECOVERY_PASS`. Aggregate execution remains +`EXECUTION_ACCEPTANCE_NOT_FULL`; no release/push/merge. Source166cda1. +Only rust_core/2/3 rolled sequentially04:24:54..04:25:46UTC to +sha256:f2040ac9e9380d067e8bcb4c965b1d7ea39c181c08993b7dba1b096337f7ca79. +OKX ingestor recreated04:26:41 at unchanged7fe34806 aftercorecatchup; real WS +snapshot generation66792 sequence17413544732 movedBNB AWAITING_SNAPSHOT->READY. +Sameconfig/TLS/resourcequotas and persistent generation; no offsets/reset/flush. +Query0ed57643,Stream65918a65,TS1037c56a,V1,alpha/order paths untouched. +Rollback3cores:7fe34806 with exact rollback.compose.json. OKXimage unchanged. + +Affected source tests:145Rustpass/1ignored,40Query/cache/SDKpass usingdisposable +Redis,clippyPASS. Oldguard counterfactual failsnewregression asintended. Optional +unchanged ignoredadmission test has1msTTL wall-clock failure, notedabove, not +included inL2passclaim. No admission policy change. Build initially used raw +imageID inFROM (Docker treatedasregistryname); corrected toverifiedlocaltag, +packagedbinaryhash checkedbeforedeploy. Counterfactualcache cleaned before +finaltests/build; no stale overlay binary deployed. + +Real TS observer300.008s:151samples/11distinctheartbeats,60/60session-ready +throughout,execution-ready49..50,zeroobservedwatermark/checksum violations, +cursorerrors0,ACKdelta30815/commitdelta17394. Telemetry complete146/151samples; +5samples44..52s had3/4reporters while allworkerheartbeatsREADY. Do notdescribe +thisascompletecounters at everyinstant. Everyrolledcontainer running,restart0, +OOMfalse. Thiswindow observedconsumer recovery; no forceddisconnect/stressrerun. + +100actualSDKreads perfeed perQueryreplica fromTScontainer (400total): +| BNB OKX read | Replica1 p50/p99 ms | Replica2 p50/p99 ms | Eligibility | +| --- | --- | --- | --- | +| BOOK_SNAPSHOT |21.55/35.19|19.64/41.44|200/200 eligible| +| BOOK_DELTA |8.65/16.66|9.24/16.72|195/200 eligible| +All400responses parsed successfully, sequenceverified/complete/no gap; +184same-watermark cross-replica metadata comparisons,0mismatch. Watermarks +advanced274768916->274779564(snapshot),274768985->274779637(delta),generation66792. +Requesttimer coversSDKcall->validatedreturn, notTSRediscommit. Sourceeventage +p99snapshot2879/2908ms; delta2679/2701ms. Five delta views at04:31:06..08 exceeded +2000ms freshness (2014..2905ms), correctly LIVE-session/STALE-event/ineligible. +Core2 contemporaneous raw-age max1686.9ms and commit max941.4ms are supporting +aggregate lag evidence, not exactperviewcausal attribution. Thisisnot recurrence +of97-minute frozenbook and notquiet-market proof. DoNOTclaim allreads executable +or widen2s. Tail-latency localization remains beforeunqualifiedexecutionFULL. + +Cleanup: testRedis/container/network removed; allprobe/build/testclients exited; +ownedbuildcontext7,344,642bytes andoldguardoverlay139,747bytes removed. Exact +BuildKit cache-idprune reclaimed0 (active-image sharedcache retained); no broad +prune orvolumesremoved. Filesystem usedbefore=206071111680, +after=205677232128bytes (concurrentactivity,notreclaimed-sizeclaim). +Activecoref2040ac9 andnamedrollback7fe34806 kept; latter also powersingestors/ +projectors. Existingbuilder/sharedtarget cache notbroad-pruned; package-scoped +counterfactual clean removed751.4MiBbeforefinalbuild. No temporaryworktreecreated. +CanonicalDL /home/bobby/data_layer,fix/live-consumer-burst-recovery; stablev2.2.0 +main8299838unchanged. TS canonical/featureworktrees unchanged; ownerplanhunks +andTScompose/symbols edits preserved. +Receipt: `/home/bobby/.local/state/qdl-v2/bnb-l2-recovery-20260928/receipt.json` +SHA256:e8f3e24bab7bb7d7bda33b78e311847a2644a1cb92999a18873bc7f1950853fa. No allcatalogue/evidence rerun; exactremaining findings areexplicit. From 59917e9b18960286ccf3979952d2b04aa9d2842f Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Mon, 28 Sep 2026 05:19:17 +0000 Subject: [PATCH 16/17] docs(plan): record approved dev-only source synchronization --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index f82dae25..915fa54a 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -60904,3 +60904,17 @@ main8299838unchanged. TS canonical/featureworktrees unchanged; ownerplanhunks andTScompose/symbols edits preserved. Receipt: `/home/bobby/.local/state/qdl-v2/bnb-l2-recovery-20260928/receipt.json` SHA256:e8f3e24bab7bb7d7bda33b78e311847a2644a1cb92999a18873bc7f1950853fa. No allcatalogue/evidence rerun; exactremaining findings areexplicit. + +### 2026-09-28 Approved Source Synchronization To Dev + +Status: IN_PROGRESS. Owner approved pushing tested patch branches and PRs to dev +only. Main, v2.2.0, runtime images, manifests and order paths remain unchanged. +Gate: reviewed CI triggers (no dev deployment), preserve owner changes, green CI +before merge, then fast-forward canonical dev and remove only verified merged +feature branches. TS local dev's ten commits are already ancestors of its consumer +branch; no duplicate cherry-pick is needed. Existing test evidence is inherited; +this source synchronization does not grant execution FULL or a new certificate. +Rollback: reviewed source revert, not runtime mutation or history rewrite. +Cleanup excludes images, state and unrelated active feature worktrees. +Guide: workspace AGENTS.md Git/Canonical Runtime rules and preceding rollout +receipt; existing BNB delta freshness limits remain explicit. From 4c6999fb5926e4fc3e666fae55271103bd2a33e8 Mon Sep 17 00:00:00 2001 From: BobbyAxerol Date: Mon, 28 Sep 2026 05:24:42 +0000 Subject: [PATCH 17/17] style(rust): format verified L2 continuity regressions --- DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md | 10 +++ rust/qdl-realtime-core/src/lib.rs | 105 +++++++++++++++------- 2 files changed, 83 insertions(+), 32 deletions(-) diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index 915fa54a..d365af89 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -60918,3 +60918,13 @@ Rollback: reviewed source revert, not runtime mutation or history rewrite. Cleanup excludes images, state and unrelated active feature worktrees. Guide: workspace AGENTS.md Git/Canonical Runtime rules and preceding rollout receipt; existing BNB delta freshness limits remain explicit. + +Source-sync CI update: PR #24 opened after push 59917e9. Contract CI failed at +Rust formatting, before clippy/tests. Applied Rust 1.82 formatter to the affected +realtime-core file; cargo fmt --all -- --check PASS in existing bounded builder, +network disabled. Formatter containers removed automatically; no image built, +no runtime changed. Initial slim-toolchain/UID attempts failed without source +changes, corrected using existing builder and host UID 1001. +TS eb6e3af pushed; API token returns 404 for ExecutorBroker (access pending). +Three inherited TS commit subjects violate governance; do not bypass that gate +or rewrite published history. Main and release remain untouched. diff --git a/rust/qdl-realtime-core/src/lib.rs b/rust/qdl-realtime-core/src/lib.rs index fe5cc214..d7b844a4 100644 --- a/rust/qdl-realtime-core/src/lib.rs +++ b/rust/qdl-realtime-core/src/lib.rs @@ -2599,7 +2599,10 @@ mod tests { "b": [["60000", bid_quantity]], "a": [["60001", "1"]]}) }; let frame = raw_with_receipt( - binding, &serde_json::to_vec(&payload).unwrap(), generation, received_at_ns, + binding, + &serde_json::to_vec(&payload).unwrap(), + generation, + received_at_ns, ); if snapshot && binding.venue == "BINANCE" { with_transport(frame, TransportProtocol::Http) @@ -2623,12 +2626,12 @@ mod tests { ]; for (offset, frame) in frames.into_iter().enumerate() { let original_receipt = frame.received_at_ns; - let expected = timely.process_at_transport_offset( - frame.clone(), original_receipt, offset as u64, - ).unwrap(); - let actual = delayed.process_at_transport_offset( - frame.clone(), now, offset as u64, - ).unwrap(); + let expected = timely + .process_at_transport_offset(frame.clone(), original_receipt, offset as u64) + .unwrap(); + let actual = delayed + .process_at_transport_offset(frame.clone(), now, offset as u64) + .unwrap(); assert!(actual.quarantines.is_empty(), "{}", binding.venue); assert_eq!(actual.canonical.len(), expected.canonical.len()); for (record, expected) in actual.canonical.iter().zip(&expected.canonical) { @@ -2641,11 +2644,21 @@ mod tests { assert_eq!(event.source_event_time_ns, original_receipt); } if offset == 1 || offset == 2 { - assert!(!actual.canonical.is_empty(), "delayed deltas cannot be skipped"); + assert!( + !actual.canonical.is_empty(), + "delayed deltas cannot be skipped" + ); } if offset == 2 { - assert!(delayed.l2_adapters[&binding.key()].core().view().unwrap().bids.is_empty()); - let duplicate = delayed.process_at_transport_offset(frame, now, offset as u64).unwrap(); + assert!(delayed.l2_adapters[&binding.key()] + .core() + .view() + .unwrap() + .bids + .is_empty()); + let duplicate = delayed + .process_at_transport_offset(frame, now, offset as u64) + .unwrap(); assert!(duplicate.canonical.is_empty()); assert_eq!(duplicate.duplicates, 1); } @@ -2669,15 +2682,21 @@ mod tests { ] { assert!(core.process(frame, now).unwrap().quarantines.is_empty()); } - let gap = core.process(delayed_book_frame( - &binding, false, 110, 109, 7, receipt, "3", - ), now).unwrap(); + let gap = core + .process( + delayed_book_frame(&binding, false, 110, 109, 7, receipt, "3"), + now, + ) + .unwrap(); assert!(gap.canonical.is_empty()); assert_eq!(gap.quarantines.len(), 1); assert!(core.l2_adapters[&binding.key()].core().view().is_none()); - let after = core.process(delayed_book_frame( - &binding, false, 111, 110, 7, now, "4", - ), now).unwrap(); + let after = core + .process( + delayed_book_frame(&binding, false, 111, 110, 7, now, "4"), + now, + ) + .unwrap(); assert!(after.canonical.is_empty()); for frame in [ delayed_book_frame(&binding, true, 200, 0, 8, now, "5"), @@ -2685,7 +2704,14 @@ mod tests { ] { assert!(core.process(frame, now).unwrap().quarantines.is_empty()); } - assert_eq!(core.l2_adapters[&binding.key()].core().view().unwrap().last_sequence, 201); + assert_eq!( + core.l2_adapters[&binding.key()] + .core() + .view() + .unwrap() + .last_sequence, + 201 + ); } } @@ -2701,15 +2727,21 @@ mod tests { ] { core.process(frame, now).unwrap(); } - let invalid = core.process(delayed_book_frame( - &binding, false, 102, 101, 7, receipt, "-1", - ), now).unwrap(); + let invalid = core + .process( + delayed_book_frame(&binding, false, 102, 101, 7, receipt, "-1"), + now, + ) + .unwrap(); assert!(invalid.canonical.is_empty()); assert_eq!(invalid.quarantines.len(), 1); assert!(core.l2_adapters[&binding.key()].core().view().is_none()); - let fresh = core.process(delayed_book_frame( - &binding, false, 103, 102, 7, now, "3", - ), now).unwrap(); + let fresh = core + .process( + delayed_book_frame(&binding, false, 103, 102, 7, now, "3"), + now, + ) + .unwrap(); assert!(fresh.canonical.is_empty()); } } @@ -2726,18 +2758,27 @@ mod tests { ] { core.process(frame, now).unwrap(); } - let unanchored = core.process(delayed_book_frame( - &binding, false, 201, 200, 8, receipt, "3", - ), now).unwrap(); + let unanchored = core + .process( + delayed_book_frame(&binding, false, 201, 200, 8, receipt, "3"), + now, + ) + .unwrap(); assert!(unanchored.canonical.is_empty()); assert!(core.l2_adapters[&binding.key()].core().view().is_none()); - let anchor = core.process(delayed_book_frame( - &binding, true, 200, 0, 8, receipt, "4", - ), now).unwrap(); + let anchor = core + .process( + delayed_book_frame(&binding, true, 200, 0, 8, receipt, "4"), + now, + ) + .unwrap(); assert!(!anchor.canonical.is_empty()); - let old = core.process(delayed_book_frame( - &binding, true, 300, 0, 7, now, "99", - ), now).unwrap(); + let old = core + .process( + delayed_book_frame(&binding, true, 300, 0, 7, now, "99"), + now, + ) + .unwrap(); assert!(old.canonical.is_empty()); assert_eq!(old.filtered_outcome, Some("IGNORED_STALE_GENERATION")); let view = core.l2_adapters[&binding.key()].core().view().unwrap();