diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5727266..f319dbe 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -119,17 +119,17 @@ jobs: cartesi-machine-sha256-arm64: ${{ env.CARTESI_MACHINE_SHA256_ARM64 }} install-foundry: "true" - - name: Install faketime + - name: Install native test dependencies run: | sudo apt-get update - sudo apt-get install -y faketime libfaketime + sudo apt-get install -y faketime libfaketime libclang-dev - name: Build watchdog Lua deps run: | sudo apt-get install -y libcurl4-openssl-dev build-essential pkg-config just watchdog-lua-deps - - name: Run rollups E2E tests + - name: Run rollups E2E tests (Rust and C hosts) run: just test-rollups-e2e # Runs after the e2e step so the canonical machine image is already built; diff --git a/AGENTS.md b/AGENTS.md index 92b5cb7..d18eab3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -197,7 +197,10 @@ Paths below are relative to `sequencer/src/`: mandatory offset with its source in `application_inputs`. - **History version** — `(EraId, RecoveryGeneration)`. Setup publishes a complete baseline with a fresh era; recovery increments the generation exactly once - iff it invalidates at least one valid batch. Subscription claims enforce both. + iff it invalidates at least one valid batch and records the preserved-prefix + cut in the same transaction. `/history` can check a saved checkpoint across + intervening generations; subscription claims still enforce both identifiers. + The [history contract](docs/protocol/application-history.md) owns compatibility. - **Soft confirmation** — sequencer's predicted ordering, emitted before the batch lands on L1. - **Snapshot** — immutable artifact at every batch close, registered with its local batch identity and application count. Acceptance facts select the diff --git a/Cargo.lock b/Cargo.lock index d0ce0bd..8c2d8d0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3875,6 +3875,8 @@ dependencies = [ "alloy-primitives", "alloy-sol-types", "app-core", + "c-app-engine", + "c-wallet-engine", "ethereum_ssz", "futures", "libtest-mimic", diff --git a/README.md b/README.md index 72a8306..515989c 100644 --- a/README.md +++ b/README.md @@ -254,6 +254,133 @@ Message shapes: { "kind": "direct_input", "offset": 11, "sender": "0x...", "block_number": 123, "block_timestamp": 1700000000, "transaction_hash": "0x...", "payload": "0x...", "input_index": 42, "batch_nonce": 4 } ``` +### History metadata and historical L1 inputs (internal only) + +Readers that maintain additional transfer/order history can reconstruct it from +L1 and then join the application feed. The +[projection replay contract](docs/protocol/projection-replay.md) describes +bootstrap, client checkpoints, pending directs, and terminal drain. + +`GET /history` returns one coherent view of the deployment, current application +history, immutable era baseline, and latest accepted checkpoint. Optional +`era_id=` requires the selected era; a mismatch returns `409 ERA_CHANGED`. +Example immediately after a rebuild: + +```json +{ + "deployment": { + "chain_id": 31337, + "app_address": "0x1111111111111111111111111111111111111111", + "input_box_address": "0x2222222222222222222222222222222222222222", + "app_deployment_block": 1, + "batch_submitter_address": "0x3333333333333333333333333333333333333333" + }, + "history": { + "version": { + "era_id": "22222222-2222-4222-8222-222222222222", + "recovery_generation": 0 + }, + "available_from": 7, + "head": 7 + }, + "baseline": { + "l1_stop_block": 1240, + "l1_end_input_index": 8, + "next_batch_nonce": 2 + }, + "accepted_checkpoint": null, + "compatibility": null +} +``` + +- `history.available_from` is baseline application count `K`; entries `[K,head)` + are available through WS. Counts include all executed application inputs. +- `baseline` describes the fixed L1 stopping block `C`, exclusive InputBox end + `R`, and scheduler nonce after recovery's terminal drain. It survives generation + changes and baseline artifact GC. It is distinct from the moving safe head. +- `accepted_checkpoint`, when available, has `inclusion_block`, + `executed_input_count`, and `next_batch_nonce`, under `history.version`. + Genesis supplies the zero checkpoint; a rebuilt baseline is not itself an + accepted checkpoint. The metadata does not lease or download a native artifact + and does not certify a client projection. A known divergence returns `503`. +- `compatibility` is `null` unless `from_generation=` is supplied together + with `era_id`. It then contains `from_generation` and `preserved_input_count`: + the prefix that survived every standard recovery since that generation, + bounded by the current head. A future generation or missing era returns + `400 BAD_REQUEST`; an era mismatch takes precedence over the generation bound. + +For example, `GET /history?era_id=&from_generation=0` can return +`"compatibility": {"from_generation": 0, "preserved_input_count": 3}`. +A saved checkpoint from that era/generation is reusable when its count `X` +satisfies `K <= X <= 3`. The boundary is inclusive: the checkpoint has executed +entries before `X`, and resumes at entry `X`. Each checkpoint must be checked +using its own saved generation. With no intervening recovery, the bound is the +current head. The [history contract](docs/protocol/application-history.md#checkpoint-compatibility-after-standard-recovery) +defines the calculation and trust boundary. + +Restore an eligible checkpoint, persist the response's current history version +with it, and subscribe using that version and its actual count. A recovery +between lookup and subscription still returns `STALE_GENERATION`; repeat the +lookup using the version associated with the restored state. Compatibility does +not certify the client's application or projection implementation, and cannot +cross a cockroach recovery's new era. + +`GET /historical-l1-inputs` requires `era_id` and exactly one starting selector: + +- `next_input_index=`: inclusive per-application InputBox index, starting at 0. +- `after_block=`: initially seek to the first input strictly after that block; + continue using the returned `next_input_index`. + +The endpoint serves only `[0,R)` through the selected era's `C`. A response to +`next_input_index=5&limit=1` can be: + +```json +{ + "era_id": "22222222-2222-4222-8222-222222222222", + "l1_stop_block": 1240, + "end_input_index": 8, + "next_input_index": 6, + "items": [{ + "input_index": 5, + "sender": "0x3333333333333333333333333333333333333333", + "payload": "0x00", + "block_number": 1230, + "block_timestamp": 1700014760, + "transaction_hash": "0x4444444444444444444444444444444444444444444444444444444444444444" + }] +} +``` + +Records preserve original inner payloads and authenticated senders, including +malformed/rejected batches; they are not complete `EvmAdvance` envelopes. Indices +are contiguous and ordered. Binary values are hex; timestamps are Unix seconds. +Clients must preserve integer precision. A page may split a block. + +Optional `limit` defaults to 256 and accepts 1–256. Pages target 1 MiB of raw +payloads; a larger first input is returned alone, intact. Hex encoding increases +wire size, so this is not a hard response-size limit. Eight historical responses +can be in flight; a permit remains held through body delivery or cancellation. +SQLite read transactions end before network delivery. These limits bound memory +by the page target or largest single input, not total history length. + +Only `next_input_index == end_input_index` means EOF; a short page does not. +Requesting `next_input_index=R` or `after_block=C` returns an empty completed page. +Generation changes do not invalidate historical pages; an era change does. + +Malformed/unknown query fields, invalid selectors/limits, or positions above +`R`/`C` return the existing `400 BAD_REQUEST` JSON shape. An era mismatch returns +the existing `409 ERA_CHANGED` history-policy body before semantic position +checks. Capacity exhaustion returns `429 OVERLOADED`; shutdown or an operational +read failure returns `503 UNAVAILABLE`. Interrupted bodies are failed pages. +Missing durable rows or other storage invariant failures follow the process's +terminal fault policy, never a successful partial page. + +The Rust SDK exposes `history(expected_era, from_generation)` and +`historical_l1_inputs(era, start, limit)` with typed metadata and era refusals. +Both use the configured request deadline, including body transfer; callers may +increase it for large historical inputs. The client owns replay, persistence, +checkpoint selection, and subscription. + ### Operator snapshot endpoints (internal only) These serve application state to the operator's watchdog and indexers. diff --git a/bindings/c-app-engine/README.md b/bindings/c-app-engine/README.md index d8ffb17..3113c8d 100644 --- a/bindings/c-app-engine/README.md +++ b/bindings/c-app-engine/README.md @@ -107,3 +107,23 @@ notices and vouchers, rejection/no-op progress, dump round trips, independent instances, and fatal/error classification. `cargo test -p c-app-engine --lib` also checks mixed-output ordering, copying reused engine buffers, and full-width voucher values with a small ABI fixture. + +The `c_host_` scenarios in `rollups-e2e` launch the reference C host with generated +devnet genesis, delete that source before startup, and exercise ordinary execution, +clean restart, stale recovery, and checkpoint-based rebuild. Independent +`EngineApp` replicas restore HTTP archives, discard the downloaded sources, +follow backlog and live WS inputs, and check history identity across recovery. +Ordinary execution and both recovery paths also compare the host against the +canonical machine. Run them with: + +```sh +just setup +just ensure-machine-image +cargo build --locked -p c-wallet-engine --bin c-wallet-genesis -p c-wallet-sequencer --bin c-wallet-sequencer -p rollups-e2e --bin rollups-e2e +target/debug/rollups-e2e c_host_ --nocapture +``` + +`just test-rollups-e2e` includes these scenarios in CI. This covers the reference +wallet across the C ABI; private engines still need their own integration and +canonical comparison. Rebuilding from a native recovery archive does not test +the application's canonical-machine-to-native exporter. diff --git a/docs/invariants.md b/docs/invariants.md index 422aaff..5a9b6c6 100644 --- a/docs/invariants.md +++ b/docs/invariants.md @@ -81,7 +81,7 @@ by writer and are write-once (`0001_schema.sql`). | inclusion lane | `batches` (insert + `sealed_at_ms`), `frames`, `user_ops`, `application_inputs`, `dumps`/`snapshots` (batch close) | | input reader | `safe_inputs`, `l1_safe_head`, `safe_accepted_batches`, `canonical_divergence` (the divergence poison marker) | | recovery (startup) | `batches.invalidated_at_ms`, Tip reopen, current `application_inputs` suffix deletion | -| history metadata (setup/recovery) | `history_state` — complete era/application-count/L1-block baseline, generation bump in a non-empty standard-recovery cascade | +| history metadata (setup/recovery) | `history_state` — complete era/application-count/L1-block baseline; generation advance and immutable preserved-prefix cut in a non-empty standard-recovery cascade | | batch submitter and mempool flusher | `wallet_nonce_watermark` — deliberately shared under one protocol: each raises it before its first broadcast (write-before-broadcast, I14) | | egress (HTTP) | `dumps.lease_count` (leases); `run`'s startup hygiene resets it to zero as the crash backstop | | setup | `deployment_identity` (pinned once), `batch_tree_anchor` (the root nonce, frozen once setup completes), the initial `dumps` + `snapshots` rows (genesis or rebuild registration, atomic with the complete history baseline), the `setup_complete` fact (written once), `batch_policy.log_gas_price` + `log_gas_price_updated_at_ms` (first write; Fixed and Uniswap) | @@ -160,7 +160,7 @@ by writer and are write-once (`0001_schema.sql`). ### I5. Recovery removes exactly the invalidated application suffix - **Holds:** invalidating a batch deletes its `application_inputs` through the - schema trigger. The cascade, generation increment, and replacement Tip commit + schema trigger. The cascade, generation cut/increment, and replacement Tip commit together. Original source records and immutable snapshots remain; snapshot selection excludes invalidated batches and GC retires their unleased artifacts. - **Enforced by:** `cascade_and_reopen`, application-input constraints, valid views. @@ -425,7 +425,10 @@ by writer and are write-once (`0001_schema.sql`). transaction. The history row is absent before this boundary. `K` and `C` remain immutable even after baseline artifact GC or recovery-root invalidation. - **Standard recovery:** one generation increment iff a valid batch is - invalidated, in the cascade transaction. Clean restart changes neither token. + invalidated, with an immutable cut at the count after suffix deletion and + before replacement directs. The entire transition commits in the cascade + transaction. Clean restart changes neither token. Every intervening cut is + required to authorize reusing a checkpoint from an older generation. - **Enforced by:** `complete_baseline_setup`, immutable history triggers, exact-`+1` generation trigger, and `cascade_and_reopen`. - **Depended on by:** mandatory snapshot-derived WS claims. Identity is validated diff --git a/docs/plans/2026-07-coordination-tracks.md b/docs/plans/2026-07-coordination-tracks.md index d10ffd4..75c8581 100644 --- a/docs/plans/2026-07-coordination-tracks.md +++ b/docs/plans/2026-07-coordination-tracks.md @@ -12,14 +12,14 @@ freely at this stage — no backward-compatibility constraints. | # | Track | Owner | Status | |---|-------|-------|--------| -| 3 | Feed & replay protocol redesign | us (design) → us/Stephen (impl) | **implemented** — canonical application history, snapshot restore archives, mandatory WS claims, typed refusals, and SDK cutover; [remaining integration gates](2026-07-track3-feed-replay-design.md#remaining-integration-gates) | +| 3 | Feed & replay protocol redesign | us (design) → us/Stephen (impl) | **repository API implemented; post-merge adoption and deployment work remain** — [follow-up sequence](2026-07-track3-feed-replay-design.md#follow-up-sequence) and [ownership](2026-07-track3-feed-replay-design.md#merge-scope-and-follow-up-ownership) | | 5 | Fee exponentiation LUT | us | **deferred** — decided exact-floor if built (the table *is* the spec, algorithm-free; replay continuity across the upgrade explicitly not preserved); a separate pending design decision may make log-space fees defunct — revisit after syncing with Bart | | 6 | Dump / `Application` API redesign | us + Bart | **interface and reference C binding implemented** — [Application contract](../protocol/application-contract.md); native-engine integration gates remain | **Current campaign order:** -1. Validate snapshot-to-live replica bootstrap through the reference C bridge, then the private DEX engine when shared. -2. Remeasure feed latency in the representative environment. +1. Merge the implemented egress API after repository review/checks. Bart can then integrate his client; adjust the API from concrete feedback without waiting for downstream completion. +2. Application integrators/operators validate the private engine, canonical-to-native exporter and recovery drill, and representative capacity before production use. Reference C-host lifecycle coverage is part of repository CI. 3. Track 5 (fee LUT) only after the log-space-fees decision. Full restore archives now support file and directory application prefixes. @@ -33,9 +33,15 @@ bootstrap, history identity, replay, and recovery boundaries. The and canonical recovery/watchdog gates have a [validation record](../review/2026-09-16-track3-validation.md). -Remaining work is native-engine integration and representative deployment -latency, tracked in the [integration plan](2026-07-track3-feed-replay-design.md). -Additional transport or retention mechanisms require a measured consumer need. +Readers whose projections contain information absent from the latest application +state can use the implemented fixed-prefix historical L1 API and checkpoint +metadata. The [projection contract](../protocol/projection-replay.md) owns that +workflow; the history contract owns implemented checkpoint compatibility across +standard recoveries. The [integration plan](2026-07-track3-feed-replay-design.md) +owns follow-up requirements and their owners. Native-engine integration and +representative deployment latency remain open after merge; they are not egress +API merge prerequisites. Other transport or retention mechanisms require a +measured need. ## Track 5 — Fee exponentiation LUT (deferred) @@ -57,17 +63,26 @@ until the pending log-space-fees decision lands (with Bart). The [Application contract](../protocol/application-contract.md) owns execution, engine progress, and checkpoint semantics. The [C binding guide](../protocol/c-application-binding.md) -maps that contract to native engines; its reference conformance suite is -implemented. End-to-end native snapshot-to-live bootstrap remains an integration -gate, alongside the private DEX engine when available. Reference bridge -conformance cannot establish private-engine correctness. +maps that contract to native engines. Its reference conformance suite and C-host +process scenarios cover snapshot-to-live bootstrap, restart, standard recovery, +and fresh-era rebuild with canonical comparison. Reference bridge conformance +cannot establish private-engine correctness. Remaining checks need the actual consumer: -- Exercise snapshot-to-live bootstrap and canonical comparison through the C - host in CI; its current smoke test builds and invokes `--help`. A reusable - conformance runner needs engine-supplied genesis and meaningful accepted and - rejected inputs. Compare canonical state files, not recovery-dump layouts. +- Supply the application's versioned canonical-machine-to-native recovery + exporter and completed operator runbook. Require the + [non-genesis recovery drill](../recovery/cockroach.md#recovery-readiness-before-deployment) + for production readiness: the old native state is unavailable, the exported + bundle restores correctly, and execution after rebuild matches the canonical + machine. For the DEX, pin the designated state drive/memory region and derive + resume metadata from canonical execution. Add the integration check to the + release validation once the actual artifacts are available; no generic trait + or deployment gate currently enforces this requirement. +- Repeat snapshot-to-live bootstrap and canonical comparison with the private + engine's genesis and meaningful accepted/rejected inputs. The reference + `c_host_` scenarios supply the lifecycle pattern; compare canonical state + files, not recovery-dump layouts. - Verify the external scheduler's ordering, fee conversion, and recovery agreement. Publish independent-port fee vectors for the [current arithmetic](../../sequencer-core/src/fee.rs); a deferred LUT is a diff --git a/docs/plans/2026-07-track3-feed-replay-design.md b/docs/plans/2026-07-track3-feed-replay-design.md index 6a2700d..31e06fe 100644 --- a/docs/plans/2026-07-track3-feed-replay-design.md +++ b/docs/plans/2026-07-track3-feed-replay-design.md @@ -8,24 +8,111 @@ The application-history protocol is implemented. Its current contracts live in: - [Snapshot lifecycle](../snapshots/lifecycle.md): durable artifacts, accepted comparisons, recovery exports, and leases. -## Remaining integration gates +## Merge scope and follow-up ownership -1. Validate the native reference adapter's snapshot-to-live replica workflow; - repeat against the private DEX bridge when available. Reference-engine - conformance does not establish private-engine correctness. -2. Measure submit-to-matching-WS-event latency in the representative deployment, - including checkpoint creation and L1 reconciliation under the supported load. +The repository delivery includes the egress contracts, storage, SDK, and reference +replay/recovery tests. Its merge criteria are review and the relevant repository +checks. Private-engine integration, operational rehearsal, and representative +capacity measurements are follow-up work; they do not block merging this API. +Merge the implementation, let consumers integrate, and adjust the API from +concrete feedback. There are no live deployments requiring compatibility. + +| Follow-up | Owner | When it is needed | +|---|---|---| +| Private DEX scheduler, indexer, and database backups | Bart / application integration | After merge, while adopting the API. Verify complete checkpoint/claim association and scheduler replay; report missing fields or awkward workflow for adjustment. | +| Canonical-to-native export and incident rehearsal | Application integration and operators, under Track 6 | Before relying on that application's recovery procedure in production. | +| Ingress latency, indexing headroom, and recovery capacity | Sequencer/application maintainers and deployment operators | Before claiming support for the target deployment workload; measure historical serving alongside ordinary traffic. | The [validation record](../review/2026-09-16-track3-validation.md) records the wallet's nonempty cold bootstrap, concurrent replay/live delivery, recovery and rebootstrap, canonical-machine gates, and local latency measurements. Those -results do not replace the consumer/environment gates above. +results do not establish private-engine conformance or target-deployment capacity. + +The reference C host has process coverage in the `c_host_` rollups E2E scenarios: +generated genesis, source-independent `EngineApp` snapshot restore, concurrent +backlog/live replay, clean restart, stale recovery, and a fresh-era rebuild. +Ordinary execution and recovery compare against the canonical machine. The +[C binding guide](../../bindings/c-app-engine/README.md#reference-wallet) owns +the commands and scope. This closes the reference-host follow-up; the private +engine and canonical-to-native exporter still require their own evidence. + +## Application projections and recovery + +Historical bootstrap and standard-recovery checkpoint reuse are implemented. +Bounded internal readers can keep additional +application-specific transfers, orders, deals, and portfolio history outside the +sequencer's application state. The client owns indexing, complete checkpoints, +and replay. The sequencer owns optimistic ordering; the scheduler remains the +canonical authority. + +Current contracts live in the [projection replay guide](../protocol/projection-replay.md) +and [README API](../../README.md#history-metadata-and-historical-l1-inputs-internal-only). +`/history` supplies deployment/baseline/current-generation metadata and a coherent +accepted checkpoint receipt. `/historical-l1-inputs` serves the immutable raw +prefix through the era's stop block, with block seek, bounded pages, and typed +era refusal. `/history` also checks a saved generation against every intervening +recovery cut. The SDK exposes both reads. Standard recovery records cuts in its +existing transaction before replacement directs are inserted. + +The [reference integration test](../../sequencer/src/integration_tests/historical_bootstrap.rs) +restores a complete wallet/projection checkpoint, replays one-record HTTP pages +through the scheduler, exercises a malformed-batch overdue drain and terminal +drain, and subscribes at nonzero `K`. It checks application state and explicit +projection order, including a same-block pending direct and the first live +input. Storage/API/SDK tests cover limits, oversized single inputs, fixed prefix +boundaries, era/generation behavior, errors, and response deadlines. This is +reference evidence, not private-engine conformance or a deployment benchmark. + +The [checkpoint compatibility test](../../sequencer/src/integration_tests/historical_bootstrap/recovery_compatibility.rs) +uses guarded recovery, complete wallet/projection backups, HTTP compatibility +lookups, and WS replay. It distinguishes equal counts from different generations, +restores the nearest eligible checkpoint after missed recoveries, and retries a +recovery between lookup and subscription. Storage tests cover cuts before +replacement directs, empty/no-op invalidation, nonzero baselines, and rollback. + +### Goals and acceptance criteria + +| Goal | Status / remaining outcome | +|---|---| +| Historical bootstrap | Implemented: replay the complete fixed prefix and join the feed at `K`, including after a nonzero rebuild. | +| Standard recovery | Implemented: choose the nearest retained checkpoint whose prefix survived every intervening generation change, then restore/resubscribe. | +| Cockroach reader recovery | Core replay/metadata workflow and reference checkpoint restore implemented; Bart's actual checkpoint preparation, incident validation, and fallback rehearsal remain. | +| Stable coordinates | Existing era/generation/application count for claims; separate InputBox indices for raw paging. Counts alone do not certify cross-era compatibility. | +| Bounded serving cost | Page/item/response bounds implemented; representative bootstrap must preserve the ingress latency target and demonstrate catch-up headroom. | +| Operational readiness | Each production application supplies its canonical-to-native exporter, runbook, and non-genesis recovery drill under Track 6. | + +### Follow-up sequence + +1. **Consumer adoption after merge.** Bart integrates accepted-boundary checkpoint + preparation and scheduler replay using the implemented API. Sequencer + maintainers address concrete feedback as it arrives; downstream completion + is not a prerequisite for repository delivery. Before production use, rehearse + identifying an unsound projection checkpoint, including one below new `K`, + and restoring an earlier trusted backup or genesis. The API does not certify + the client's projection. +2. **Deployment readiness.** Validate the private native engine and measure + latency, historical serving cost, projection throughput, + catch-up headroom, and recovery time. Track 6 independently owns the versioned + canonical-machine exporter and native-state-unavailable drill. + +### Scope boundaries + +Keep the existing recovery terminal drain: the first accepted resumed frame +accounts for old pending directs before its user ops. Replacing that mechanism +would require carrying pending work across the baseline and is not justified by +this consumer requirement. + +Keep manual cross-era trust selection. Execution-prefix hash chains can identify +an input trace but cannot prove that an engine or indexer computed correct state; +revisit only if automated matching or measured reconstruction costs justify them. +Server-side scheduler replay producing a flattened execution archive and +submitter/key rotation remain separate future work. No client checkpoint +registration, server-side projection storage, or historical execution archive is +required by this design. ## Revisit only with a consumer need - Resumable snapshot transfer: when artifact size makes interrupted downloads costly. -- Retained client checkpoints: when full rebootstrap cost matters. -- Archival HTTP replay or raw L1 feeds: for an identified consumer. - Session fencing: if history can mutate within an admitted process or multiple local writers become supported. diff --git a/docs/plans/2026-08-authority-boundary-adr.md b/docs/plans/2026-08-authority-boundary-adr.md index b2e8672..bc1b4ce 100644 --- a/docs/plans/2026-08-authority-boundary-adr.md +++ b/docs/plans/2026-08-authority-boundary-adr.md @@ -179,7 +179,7 @@ clean restart changes neither. The pair is an equality/discontinuity token, not an ordered counter. Snapshot headers and mandatory WS claims expose these coordinates. Every application row has its pre-execution count; recovery replaces only the current suffix. See the [history contract](../protocol/application-history.md) and -[remaining integration gates](2026-07-track3-feed-replay-design.md). +[follow-up plan](2026-07-track3-feed-replay-design.md). ## Performance posture @@ -189,5 +189,5 @@ the evaluation conditions. The [retained comparison](../review/2026-09-16-track3 records exact revisions, workload, and same-host ACK/WS measurements. They are regression evidence: client/host contention and excluded startup or backlog work prevent interpreting them as deployment capacity. Representative latency, -including checkpoint and L1-reconciliation overlap, remains an -[integration gate](2026-07-track3-feed-replay-design.md#remaining-integration-gates). +including checkpoint and L1-reconciliation overlap, remains a +[deployment follow-up](2026-07-track3-feed-replay-design.md#merge-scope-and-follow-up-ownership). diff --git a/docs/protocol/application-contract.md b/docs/protocol/application-contract.md index 536a69a..3bc28a2 100644 --- a/docs/protocol/application-contract.md +++ b/docs/protocol/application-contract.md @@ -193,3 +193,30 @@ implementation. `CanonicalState::canonical_snapshot_bytes` is a separate inspection trait required by the shared Rust scheduler's inspection method and canonical harness, not by the native sequencer. Human-readable debugging state also stays on the concrete application. + +### 7. Canonical recovery integration + +Every supported production application must provide a reproducible mapping from +a trusted canonical machine checkpoint and pinned deployment configuration to +its native recovery artifact. All logical state needed for future application +execution, including progress, must be recoverable this way. Native caches and +backing resources may be reconstructed; indispensable mutable application state +cannot exist only in the sequencer's local storage. + +For the native DEX integration, the native application state is the designated +canonical machine drive/memory region. The integration must pin its location, +layout, and extraction procedure for each supported image. Other applications +may require a different mapping. The recovery bundle also needs the exact L1 +boundary and next scheduler nonce, obtained from trusted canonical execution; +these are separate from merely extracting application bytes. + +Each integration supplies a versioned export command and operator procedure, +and demonstrates recovery from a non-genesis canonical checkpoint before +production use. The [recovery readiness requirements](../recovery/cockroach.md#recovery-readiness-before-deployment) +own the procedure and drill. This is an integration/release requirement, not a +claim that the private engine or current tooling has passed it. + +Canonical extraction belongs to application-specific recovery tooling. The +runtime `Application` trait stays independent of machine image layouts and host +export tooling; requiring a method to compile would establish its availability, +not the correctness or operational readiness of the recovery path. diff --git a/docs/protocol/application-history.md b/docs/protocol/application-history.md index 2023241..ed3e3bf 100644 --- a/docs/protocol/application-history.md +++ b/docs/protocol/application-history.md @@ -55,7 +55,9 @@ If the era baseline count is `K` and the current head is `H`, available entries occupy `[K, H)`. A claim at `H` waits for future entries; a claim below `K` or above `H` is refused. Identity is checked before position. Equal counts cannot authorize resuming a different era or generation, even if the consumer believes -its state precedes the replaced suffix. +its state precedes the replaced suffix. The compatibility query below can +authorize rebinding a surviving checkpoint to the current version; WS itself +continues to require exact identity. Automatic recovery invalidates a batch suffix, removes its current application rows, advances the generation, and opens the replacement Tip in one transaction. @@ -65,6 +67,40 @@ sequence is not separately retained. A repair that invalidates nothing leaves the generation unchanged. The [recovery guide](../recovery/README.md) owns repair selection and guards. +### Checkpoint compatibility after standard recovery + +Each generation transition records the surviving application count after the +invalidated rows are removed and before the replacement Tip adds any directs. +This cut commits atomically with invalidation, the generation advance, and +reopening. An invalidated empty batch still creates a transition with the old +head as its cut; a repair that invalidates nothing creates neither. The cuts are +immutable and retained for the era's lifetime. + +For a checkpoint saved in generation `g`, `/history` returns the current version +at generation `G`, head `H`, and preserved count: + +```text +P = min(H, cut[g+1], ..., cut[G]) +``` + +For `g = G`, `P = H`. A checkpoint at count `X` is eligible to resume under the +returned version exactly when `K <= X <= P`. Check each saved checkpoint using +its own era and generation, then choose the newest eligible one. For cuts +`0 -> 1: 3` and `1 -> 2: 5`, a generation-0 checkpoint at 4 is invalid, while a +generation-1 checkpoint at 4 is eligible. Looking only at the latest cut would +incorrectly reuse the former. Cuts and current history are read together; a +missing intervening transition is an invariant failure, not permission to take +the minimum over an incomplete ledger. + +The client owns checkpoint consistency: application state, projection, and claim +must describe the same executed prefix. Once compatibility is established, +persist the new version with the restored checkpoint before continuing. If +another recovery wins the race with subscription, query again using that saved +version. A stale response cannot weaken WS admission. The query proves prefix +preservation under standard recovery's trusted local bookkeeping; it does not +inspect client state or establish trust after a software bug. A new era requires +the [manual projection recovery procedure](projection-replay.md#client-checkpoints). + ### Era baseline Setup publishes a complete baseline only after its artifact is durable: @@ -81,13 +117,14 @@ Genesis supplies the trusted block-zero comparison state. The [rebuild guide](../recovery/cockroach.md) owns checkpoint requirements and the fixed stopping boundary. -## Three consumers of checkpoints +## Consumers of checkpoints | Consumer | Starting point and continuation | |---|---| | Native restart | Load the newest surviving batch snapshot, or baseline, check the engine's count against its row, then replay current application inputs. | | Sequencer replica | Download `/latest_snapshot`, restore its application state, and subscribe using the matching history claim. This follows optimistic execution. | | Watchdog | Start from independently trusted canonical machine state and replay L1. Compare at the sequencer's accepted checkpoint; the replica feed does not establish independent trust. | +| Application projection | Reconstruct additional transfer/order history using the era's historical L1 prefix, then join the application feed at the immutable baseline. The [projection replay contract](projection-replay.md) owns its checkpoint preparation and handoff. | A batch-close snapshot is identified by its local batch identity, not just its count or nonce. Recovery can reuse a nonce and empty batches can repeat a count. @@ -109,19 +146,22 @@ explains block-boundary comparison and rollback retention. The 3. Subscribe with the matching era, generation, and next-input count. Snapshot selection, headers, and lease share one transaction; recovery during the download can still invalidate the claim before subscription. Rebootstrap - if the server refuses that old identity. + if the server refuses that old era; within the same era, a compatible saved + checkpoint can instead be selected through `/history`. 4. Require each entry's offset to equal the application's current count, then execute it through the shared execution boundary. Successful application advances the count by one. Persist the history identity with the replica's state so that a later resume cannot combine different histories. 5. After an ordinary disconnect, reconnect with that saved identity and the - actual count. A history mismatch or unavailable prefix requires a current - snapshot. A count ahead of the server's head is an invalid claim to correct. + actual count. A generation mismatch permits the compatibility procedure + above. An unavailable prefix or lack of a compatible checkpoint requires a + current snapshot. A count ahead of the server's head is an invalid claim to correct. The [Rust SDK](../../sdk/rust-client/src/lib.rs) returns a `HistoryClaim` with its snapshot response and requires an explicit claim for subscriptions. The consumer owns restore, persistence, and reconnect. Fetching fresh identity -metadata cannot authorize old application state. +metadata alone cannot authorize old application state; an explicit compatibility +result can authorize a saved prefix within the same era. One durable page reader handles both backlog and live delivery, so there is no separate cursor to switch at the live boundary. Each page reads identity, @@ -144,6 +184,7 @@ writers become supported. | Coherent application pages | [`storage/egress/canonical.rs`](../../sequencer/src/storage/egress/canonical.rs) and its tests — source context, nonzero baselines, replacement offsets, concurrent recovery, gaps and SQL limits. | | Replay followed by live delivery | [`l2_tx_feed`](../../sequencer/src/egress/l2_tx_feed/) — bounded deep replay, identity refusals, shutdown and persistent faults; [`catch_up.rs`](../../sequencer/src/ingress/inclusion_lane/catch_up.rs) for native replay. | | Artifact and claim association | [`snapshot_endpoints.rs`](../../sequencer/src/integration_tests/snapshot_endpoints.rs) — headers, restore, archive contents, and lease lifetime. | +| Checkpoint compatibility | [`storage/history.rs`](../../sequencer/src/storage/history.rs) and recovery tests — immutable cuts, complete lineage, and transaction rollback; [`recovery_compatibility.rs`](../../sequencer/src/integration_tests/historical_bootstrap/recovery_compatibility.rs) — saved projections across missed recoveries and HTTP lookup/WS admission races. | The [integration validation record](../review/2026-09-16-track3-validation.md) records wallet replica and canonical-machine evidence. Remaining consumer and diff --git a/docs/protocol/c-application-binding.md b/docs/protocol/c-application-binding.md index 4960464..49b03f1 100644 --- a/docs/protocol/c-application-binding.md +++ b/docs/protocol/c-application-binding.md @@ -49,5 +49,8 @@ The same implementation may be compiled for native execution and the canonical machine. This does not establish equivalent behavior across targets: the application must preserve deterministic state and output bytes, including the checkpoint's canonical comparison file. Reference wallet ABI tests exercise -the host integration; they do not establish private DEX conformance or -equivalence between native and machine execution. +the adapter contract; the `c_host_` process scenarios exercise snapshot replication, +restart, both recovery paths, and canonical-machine comparison as described in +the [build guide](../../bindings/c-app-engine/README.md#reference-wallet). +These tests establish reference-wallet agreement for their workloads, not +private DEX conformance. diff --git a/docs/protocol/projection-replay.md b/docs/protocol/projection-replay.md new file mode 100644 index 0000000..33f582f --- /dev/null +++ b/docs/protocol/projection-replay.md @@ -0,0 +1,122 @@ +# Application projections from historical L1 inputs + +A reader may maintain transfers, orders, or other application-specific history +absent from the latest application state. It owns that projection and its +checkpoints. The sequencer supplies raw scheduler inputs for the immutable era +baseline, then application inputs for the optimistic suffix. The canonical +scheduler remains the ordering authority. The [README API](../../README.md#history-metadata-and-historical-l1-inputs-internal-only) +owns routes, fields, limits, and refusals; the +[history contract](application-history.md) owns application coordinates. + +## Bootstrap and handoff + +1. Read `/history`. Pin the deployment identity and era, baseline application + count `K`, L1 stop block `C`, exclusive raw end `R`, and next scheduler nonce. + Use the deployment's application/genesis configuration and scheduler release, + including its batch codec, wait bound, and signing-domain name/version. +2. From trusted genesis, request historical records from raw index 0 and process + every record through the scheduler. Preserve InputBox order, including within + blocks. Malformed/rejected batch payloads still reach the scheduler: its + overdue-direct backstop runs before batch decoding. +3. Continue using the returned raw cursor until it equals `R`; page length and + block changes are not EOF. Drain the remaining directs through `C`, reproducing + the recovery terminal drain. Verify resulting count `K` and next nonce. +4. Refresh `/history?era_id=` for the current generation, confirm + the baseline association, and subscribe at application count `K`. The feed + contains entries `[K,head)` and then future entries. It does not redeliver the + inputs already included in the baseline. + +Raw records carry original inner payloads and authenticated sender/block context, +not the complete machine transport envelope. The signing domain uses the pinned +release plus deployment chain id and app address. Timestamps and transaction +hashes are provenance, not additional scheduler transition inputs. + +An ordinary generation change leaves `C/R/K/nonce` and historical pages intact. +If it occurs before subscription, refresh the same era's metadata and retry the +claim. An era change requires establishing correspondence with the new baseline; +never silently splice its pages into an earlier replay. Count/nonce equality is +a consistency check, not independent proof that a client computed correct state. + +The terminal-drained baseline need not equal canonical state at block `C`: +young directs may execute preemptively. A later accepted batch provides the +canonical comparison boundary. Keep that distinction when validating recovery. + +## Client checkpoints + +Save core application state, projection, and their actual `HistoryClaim` +consistently. An exact era-baseline checkpoint can be rebound to the current +generation after confirming the same immutable era/baseline. For suffix +checkpoints, query `/history?era_id=&from_generation=` +and require `K <= saved_count <= compatibility.preserved_input_count`. Query each +candidate using its own generation and choose the newest eligible backup. +Restore the complete application/projection checkpoint, persist the returned +version with it, and resume at its saved count. Repeat compatibility lookup if +another recovery causes WS to refuse that version. The +[history contract](application-history.md#checkpoint-compatibility-after-standard-recovery) +owns the calculation and its standard-recovery trust assumptions. + +For efficient manual recovery, prepare checkpoints at supported accepted L1 +block boundaries. Read the coherent `accepted_checkpoint` and `history.version` +from `/history`; restore an earlier compatible client backup and replay the +application feed exactly to the accepted count. Save the complete result with +that receipt. If the live reader is already ahead, use an independent restore; +the client owns the copy/replay cost. Bind the receipt to the matching history +and count. Count alone does not identify its block/nonce: empty batches can +share a count, and generations can reuse replaced offsets. + +Such a backup contains core state and projection at count `X`, inclusion block +`B`, next scheduler nonce `N`, and the application's own clock `A`. The +[manual recovery contract](../recovery/cockroach.md#replay-boundaries) requires +`A < B`, except known empty genesis, and `B <= C` for the target rebuild: + +1. Independently establish trust in the backup, projection implementation, and + checkpoint boundary under the [incident playbook](../recovery/cockroach.md#application-specific-reader-state). +2. Fetch raw records **after `A`**, not after `B`. Enqueue external directs from + `(A,B]` without executing them; skip batch envelopes in this seed range. +3. Process all records in `(B,C]` through the scheduler at nonce `N`, then perform + the same terminal drain and handoff as genesis bootstrap. + +Seeds and replay may share one paginated traversal. Page boundaries can split +block `B`; they must not cause premature replay or draining. An arbitrary +mid-batch optimistic checkpoint lacks this scheduler continuation. If an eligible +backup cannot be trusted, use an earlier eligible backup or trusted genesis. +Watchdog agreement on current application bytes does not certify the reader's +additional transfer/order history. + +If checkpointing during raw replay, persist the scheduler queue/nonce and raw +cursor alongside app/projection state, or resume from the prepared backup. +Persisting only an application count cannot resume an interrupted scheduler. + +## Worked recovery boundary + +With the 1200-block wait bound, consider valid direct inputs `D*` and accepted +user operations `U*` in this raw order: + +| Raw index | Block | Input | Application execution | +|---:|---:|---|---| +| 0 | 5 | `D0` | Queued | +| 1 | 12 | `D1` | Queued | +| 2 | 20 | Batch 0, safe block 10, `U0` | `D0`, `U0` | +| 3 | 20 | `D2`, after the batch | Queued | +| 4 | 24 | `D3` | Queued | +| 5 | 1230 | Malformed batch | Backstop executes `D1,D2,D3`; decoding rejects | +| 6 | 1232 | Batch 1, safe block 1228, `U1` | `U1` | +| 7 | 1235 | `D4` | Queued | + +The backup at `B=20` has count 2, clock `A=10`, and next nonce 1. A count-1 +backup sits partway through batch 0; it must execute `U0` before attaching this +receipt. Seed reconstruction starts at raw index 1, preserving both `D1` and the +same-block `D2` without executing batch 0 twice. + +For a rebuilt stop `C=1240`, the malformed input executes three overdue directs +without consuming a batch nonce. Batch 1 executes `U1`; terminal drain executes +`D4`. The resulting baseline is `K=7`, next nonce 2, raw end `R=8`, and app clock +1235. Subscribe at application offset 7; the next raw index is independently 8. + +The [reference integration test](../../sequencer/src/integration_tests/historical_bootstrap.rs) +restores a wallet checkpoint with separately saved notice history, deletes the +source backup, fetches one-record HTTP pages through the SDK, executes this +trace, and joins the real WS feed at nonzero `K`. It checks full wallet state and +explicit projection order against uninterrupted execution. It uses controlled +storage fixtures and a trusted checkpoint receipt; it does not establish Bart's +private database backup procedure or canonical-machine export conformance. diff --git a/docs/recovery/README.md b/docs/recovery/README.md index a1bdf1e..1a45fa5 100644 --- a/docs/recovery/README.md +++ b/docs/recovery/README.md @@ -223,7 +223,13 @@ wall-clock aging alone can change the decision after inspection. Invalidation, history rewind, generation change, and Tip creation share one transaction. Invalidation removes the suffix's `application_inputs` projection; raw source facts remain. `RecoveryGeneration` increments once iff at least one -valid batch was invalidated. Failed reopening rolls all of this back. +valid batch was invalidated. Each advance appends an immutable generation cut: +the surviving count after suffix deletion, before reopening can insert any +replacement directs. Failed reopening rolls all of this back. The cuts let +readers determine whether a saved checkpoint survived several recoveries; +the [history contract](../protocol/application-history.md#checkpoint-compatibility-after-standard-recovery) +owns that query. Invalidating an empty batch records the old head; a no-op repair +records no transition. The new Tip follows the latest surviving batch, or uses the immutable root anchor if none survives. It attributes direct inputs after the surviving diff --git a/docs/recovery/cockroach.md b/docs/recovery/cockroach.md index ff588f7..38d07d1 100644 --- a/docs/recovery/cockroach.md +++ b/docs/recovery/cockroach.md @@ -30,13 +30,125 @@ comparison checkpoint at that block. [Standard recovery](README.md) instead uses the existing database to repair an optimistic suffix automatically. It assumes the local bookkeeping is trustworthy. +## Recovery readiness before deployment + +Canonical-to-native recovery is a required +[application integration capability](../protocol/application-contract.md#7-canonical-recovery-integration). +Each production deployment must have a release-matched export command, a +completed application-specific runbook, and a successful recovery rehearsal. +An incident is an execution of that prepared procedure, not the first attempt +to determine a state layout or assemble recovery metadata. + +The application runbook must name: + +- The canonical image, native engine, state layout, exporter, and tool versions. +- How to select and preserve a trusted CM checkpoint and its exact L1 boundary. +- The command extracting the application state, count/clock, and next scheduler + nonce into the bundle accepted by `setup --recovery`, including supported + checkpoint boundaries and the loader's `A < B` requirement below. +- Artifact locations, access/backup procedures, validation commands, and the + commands to rebuild, restart, compare, and resume affected readers. +- A rehearsed fallback to an earlier trusted checkpoint or genesis if the + preferred artifact is unavailable, with measured replay time and disk needs. + +The qualifying drill starts with a **non-genesis canonical machine checkpoint**, +pinned deployment data, and L1 access, while the old native database and dumps +are unavailable. Run the actual exporter and restore its output; check the +native application bytes/progress and scheduler nonce against the canonical +source. Exercise directs pending at the checkpoint and inputs arriving after it. +Run `setup --recovery` in a fresh directory, resume sequencing, and compare +against independent canonical execution after a new batch is accepted. The +terminal-drained baseline itself need not equal canonical state at `C`. + +Automate this integration check where the artifacts are available, and require +passing evidence for the supported release before production deployment. Record +artifact versions, commands, checks, and timings; repeat when the state mapping, +checkpoint/recovery behavior, or relevant release artifacts change. A native +dump round-trip or a test using shared scheduler fixtures alone does not exercise +the canonical-machine export boundary. The +[Track 6 integration plan](../plans/2026-07-coordination-tracks.md#track-6--dump--application-api-redesign) +tracks the remaining tooling and validation work; this requirement is not an +implemented deployment gate. + +## Incident playbook + +The objective is a usable, independently trusted starting state. A newer sound +checkpoint reduces replay work; finding the exact first bad execution or the +latest possible sound checkpoint is not a prerequisite for recovery. + +1. **Stop and preserve.** Stop the sequencer and prevent automatic restarts. + Preserve its data directory, logs, and available archives before rebuilding. + Pause watchdog ticks while copying its selected checkpoint, manifest, + `head.json`, and configuration. Preserve affected client databases and their + checkpoint metadata separately. +2. **Establish the cause and reference.** Check deployment identity, canonical + machine image, bootstrap boundary, and the reported comparison boundary. + A configuration mismatch is different from faulty execution. Fix the cause + before running the replacement sequencer; retain an independently trusted + canonical machine or earlier checkpoint as the reference. +3. **Select a sound application checkpoint.** The watchdog's durable head is + its last successful comparison checkpoint, or its operator-trusted initial + checkpoint if no comparison succeeded. A failed comparison does not replace + it; initialization and idle ticks are not successful comparisons. Use that + canonical state, or independently validate a retained candidate at the same + exact L1 boundary. Current-state equality can establish a usable application + state without establishing that all earlier executions were correct. +4. **Prepare a restorable native bundle.** Validate the candidate's restored + application state, embedded count/clock, and next batch nonce against the + canonical reference at block `B`. Keep the artifact and its boundary metadata + together and record how its trust was established. The receipt alone is not + evidence that a faulty sequencer executed correctly. +5. **Rebuild in a fresh directory.** Use the invocation below. Recovery chooses + its post-flush stopping block `C`, replays from the trusted checkpoint, and + publishes a new era. Preserve that baseline artifact and its metadata for + client alignment before resumed operation can collect it. Resume independent + watchdog comparison when a new accepted comparison checkpoint is available. + +### Obtaining the recovery artifact + +The watchdog stores a whole CM, including scheduler state; it does not save a +native `/finalized_snapshot` archive. Use the application's rehearsed canonical +export command to obtain the native bundle, or a retained native archive whose +state and resume metadata can be validated against the canonical reference. +The mapping is required even when it is a direct extraction of a designated +drive. The generic watchdog does not implement that application-specific command. +A comparison file alone need not contain everything an engine requires to restore. + +Retain verified native archives outside sequencer GC if they are the intended +recovery source. The watchdog normally prunes its previous CM checkpoint, and +sequencer GC may remove the native artifact from the last passing comparison +after a newer batch is accepted. Downloading `/finalized_snapshot` after an +alarm can return the faulty newer state. The +[backup guide](../watchdog/operator-deployment.md#checkpoint-disk-usage-and-backups) +describes retention; the application runbook supplies the tested conversion. +Use its rehearsed earlier-checkpoint/genesis fallback when the preferred source +is unavailable. + +### Application-specific reader state + +A reader such as Bart's indexer owns more state than the sequencer application. +Choose its latest checkpoint whose execution provenance and indexing behavior +remain trustworthy after diagnosing the incident. Its boundary may differ from +the sequencer's chosen checkpoint. Compare against an independent reconstruction +of the required projection when needed; matching current balances or positions +does not validate historical transfers, deals, or portfolio records. + +Restore that complete client checkpoint and its scheduler continuation metadata, +then replay canonical L1 inputs through the replacement era's `C` and perform +the terminal drain. Establish agreement with the replacement application +baseline, including count `K`, before binding the reader to the new history +claim. If no client checkpoint can be trusted, rebuild its projection from a +trusted origin. A current application snapshot cannot supply omitted history. + +The [projection replay contract](../protocol/projection-replay.md) describes the +historical-input API, checkpoint preparation, and handoff metadata. Its reference +test exercises a trusted client checkpoint; the application-specific backup and +incident validation remain integration work. This manual procedure does not +require automated cross-era checkpoint matching. + ## Run a rebuild -Stop the old sequencer and resolve the cause of the failure. Choose a trusted -canonical application checkpoint; a recent one reduces replay work. Its state, -inclusion block, and next batch nonce must agree. After a bug, establish that -trust independently of the faulty local state, using a trusted canonical-machine -checkpoint or replay from a trusted origin. +Select and prepare the trusted checkpoint using the incident playbook above. The loader requires an application artifact, `info.toml`, and `checkpoint.toml`. The [recovery export workflow](../snapshots/lifecycle.md#http-and-recovery-exports) diff --git a/docs/review/register.md b/docs/review/register.md index 90c9713..238f252 100644 --- a/docs/review/register.md +++ b/docs/review/register.md @@ -51,6 +51,13 @@ exposure in an actual deployment was established by this review. ## Bounded investigations and cleanup +- **Intermittent process-lock test failure.** The macOS workspace suite can + report `Locked` at the final reacquisition in + `dropped_runtime_scope_keeps_lock_until_detached_worker_stops` in + [`workers.rs`](../../sequencer/src/commands/run/workers.rs); an isolated rerun + passes. The worker drops its scope before signalling completion, so a simple + worker-completion race does not explain the failure. Identify any remaining + descriptor/process ownership before changing the assertion or lock behavior. - **Transient SQLite contention stops the submitter.** Read handles use a 50 ms busy timeout; a storage/open failure escapes the submitter loop. BUSY/LOCKED are nonterminal but project to unclassified exit 1, causing @@ -111,11 +118,11 @@ replay; no new snapshot lifecycle is implied. ## Integration work owned elsewhere -- [Track 3 integration gates](../plans/2026-07-track3-feed-replay-design.md): - native snapshot-to-live replication and representative deployment latency, +- [Track 3 follow-ups](../plans/2026-07-track3-feed-replay-design.md): + private-engine snapshot-to-live replication and representative deployment latency, including checkpoint creation and complete L1-reconciliation turns. - [Track 6](../plans/2026-07-coordination-tracks.md#track-6--dump--application-api-redesign): - external engine/scheduler agreement, C-host end-to-end coverage, independent + external engine/scheduler agreement, application-specific recovery export, independent fee-conversion vectors, and consumer-driven ABI/checkpoint decisions. The [2026-09-16 validation record](2026-09-16-track3-validation.md) supports diff --git a/docs/watchdog/operator-deployment.md b/docs/watchdog/operator-deployment.md index f9b31d8..e7c3c7d 100644 --- a/docs/watchdog/operator-deployment.md +++ b/docs/watchdog/operator-deployment.md @@ -381,6 +381,15 @@ checkpoint as that archive. The [snapshot backup workflow](../snapshots/lifecycle.md#http-and-recovery-exports) owns this separate restore path. +The [incident playbook](../recovery/cockroach.md#incident-playbook) covers +selecting a sound state, validating a native recovery bundle, and recovering +application-specific reader databases. It also identifies the CM-to-native +export and retention requirements that operator backups must cover. +Complete the application's +[recovery readiness procedure and drill](../recovery/cockroach.md#recovery-readiness-before-deployment) +before production deployment; a retained CM checkpoint is useful only with a +known, tested path back to a running native application. + ## Sequencer restart policy The sequencer's exit codes are the restart contract: 10 @@ -415,12 +424,13 @@ unclassified restart-with-backoff. Operational notes: also logs the latest row once at startup). Any death that did not return through the bracket — SIGKILL, OOM, a node reboot, a terminal runtime abort, a controller panic — leaves only the process logs. -- **Canonical divergence is the one manual path**: the sequencer freezes - the acceptance frontier - ([I15](../invariants.md#i15-divergence-marker-present--acceptance-frontier-frozen)), - refuses every command on that data directory, and the remedy is a - fresh-directory `setup --recovery` (cockroach). You will typically learn - of it from the watchdog before the sequencer tells you. +- **Untrustworthy local state requires manual recovery.** The reader's + content-identity divergence marker freezes the acceptance frontier and + blocks admission ([I15](../invariants.md#i15-divergence-marker-present--acceptance-frontier-frozen)). + A watchdog state mismatch independently signals application-state + disagreement. Diagnose the cause and follow the + [incident playbook](../recovery/cockroach.md#incident-playbook) for a + fresh-directory rebuild when needed; standard recovery does not repair bugs. ## Troubleshooting (live deployments) diff --git a/justfile b/justfile index 91ccdfe..2b98519 100644 --- a/justfile +++ b/justfile @@ -65,7 +65,7 @@ test-sequencer: test-rollups-e2e: setup ensure-machine-image ensure-sepolia-machine-image just watchdog-lua-deps - cargo build -p wallet-sequencer --bin wallet-sequencer-devnet -p rollups-e2e --bin rollups-e2e + cargo build -p wallet-sequencer --bin wallet-sequencer-devnet -p c-wallet-engine --bin c-wallet-genesis -p c-wallet-sequencer --bin c-wallet-sequencer -p rollups-e2e --bin rollups-e2e cargo run -p rollups-e2e --bin rollups-e2e ensure-machine-image: diff --git a/sdk/rust-client/src/errors.rs b/sdk/rust-client/src/errors.rs index ec35b32..cd7ed73 100644 --- a/sdk/rust-client/src/errors.rs +++ b/sdk/rust-client/src/errors.rs @@ -84,3 +84,15 @@ pub enum SnapshotError { #[error("invalid snapshot metadata: {0}")] Metadata(String), } + +#[derive(Debug, Error)] +pub enum HistoryReadError { + #[error("history request failed: {0}")] + Request(#[from] reqwest::Error), + #[error(transparent)] + History(#[from] sequencer_core::history::HistoryPolicyError), + #[error("history request rejected with status {status}: {body}")] + Http { status: u16, body: String }, + #[error("invalid history response: {0}")] + Decode(#[from] serde_json::Error), +} diff --git a/sdk/rust-client/src/history.rs b/sdk/rust-client/src/history.rs new file mode 100644 index 0000000..35a5da2 --- /dev/null +++ b/sdk/rust-client/src/history.rs @@ -0,0 +1,359 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +use crate::{ + EraId, HistoricalL1InputStart, HistoricalL1InputsPage, HistoryInfo, HistoryPolicyError, + HistoryReadError, RecoveryGeneration, SequencerClient, +}; + +impl SequencerClient { + /// Discover history, optionally requiring the era selected for bootstrap. + /// A compatibility query supplies both the checkpoint's era and its generation. + pub async fn history( + &self, + expected_era: Option, + from_generation: Option, + ) -> Result { + let mut query: Vec<_> = expected_era + .map(|era| ("era_id", era.to_string())) + .into_iter() + .collect(); + if let Some(generation) = from_generation { + query.push(("from_generation", generation.get().to_string())); + } + let request = self + .http_client + .get(format!("{}/history", self.endpoint.trim_end_matches('/'))) + .query(&query) + .timeout(self.request_timeout); + let body = read_history_response(request).await?; + Ok(serde_json::from_str(&body)?) + } + + /// Read one complete raw-input page under the configured request deadline. + /// Continue with its `next_input_index`; a short page is not necessarily EOF. + /// Use `with_request_timeout` when historical transfers need a longer deadline. + pub async fn historical_l1_inputs( + &self, + era: EraId, + start: HistoricalL1InputStart, + limit: Option, + ) -> Result { + let mut query = vec![("era_id", era.to_string())]; + query.push(match start { + HistoricalL1InputStart::NextInputIndex(index) => { + ("next_input_index", index.to_string()) + } + HistoricalL1InputStart::AfterBlock(block) => ("after_block", block.to_string()), + }); + if let Some(limit) = limit { + query.push(("limit", limit.to_string())); + } + let request = self + .http_client + .get(format!( + "{}/historical-l1-inputs", + self.endpoint.trim_end_matches('/') + )) + .query(&query) + .timeout(self.request_timeout); + let body = read_history_response(request).await?; + Ok(serde_json::from_str(&body)?) + } +} + +async fn read_history_response( + request: reqwest::RequestBuilder, +) -> Result { + let response = request.send().await?; + let status = response.status().as_u16(); + let body = response.text().await?; + if status == 409 + && let Ok(policy) = serde_json::from_str::(&body) + { + return Err(HistoryReadError::History(policy)); + } + if status != 200 { + return Err(HistoryReadError::Http { status, body }); + } + Ok(body) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::time::Duration; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + use tokio::net::{TcpListener, TcpStream}; + use tokio::task::JoinHandle; + + const ERA: &str = "00112233-4455-4677-8899-aabbccddeeff"; + + async fn request_headers(stream: &mut TcpStream) -> String { + let mut headers = Vec::new(); + while !headers.ends_with(b"\r\n\r\n") { + let mut byte = [0]; + assert_eq!(stream.read(&mut byte).await.unwrap(), 1); + headers.push(byte[0]); + assert!(headers.len() <= 8192, "request headers exceed test bound"); + } + String::from_utf8(headers).unwrap() + } + + async fn serve_once(status: u16, body: String) -> (SequencerClient, JoinHandle) { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let endpoint = format!("http://{}/", listener.local_addr().unwrap()); + let server = tokio::spawn(async move { + let (mut stream, _) = listener.accept().await.unwrap(); + let headers = request_headers(&mut stream).await; + stream + .write_all( + format!( + "HTTP/1.1 {status} Test\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", + body.len() + ) + .as_bytes(), + ) + .await + .unwrap(); + headers + }); + (SequencerClient::new(endpoint).unwrap(), server) + } + + fn request_url(headers: &str) -> reqwest::Url { + let target = headers + .lines() + .next() + .unwrap() + .split_whitespace() + .nth(1) + .unwrap(); + reqwest::Url::parse(&format!("http://localhost{target}")).unwrap() + } + + fn history_body() -> String { + serde_json::json!({ + "deployment": { + "chain_id": 31337, + "app_address": "0x1111111111111111111111111111111111111111", + "input_box_address": "0x2222222222222222222222222222222222222222", + "app_deployment_block": 1, + "batch_submitter_address": "0x3333333333333333333333333333333333333333" + }, + "history": { + "version": { "era_id": ERA, "recovery_generation": 2 }, + "available_from": 7, + "head": 12 + }, + "baseline": { + "l1_stop_block": 1240, + "l1_end_input_index": 8, + "next_batch_nonce": 2 + }, + "accepted_checkpoint": { + "inclusion_block": 1250, + "executed_input_count": 10, + "next_batch_nonce": 3 + }, + "compatibility": null + }) + .to_string() + } + + fn page_body() -> String { + serde_json::json!({ + "era_id": ERA, + "l1_stop_block": 1240, + "end_input_index": 8, + "next_input_index": 6, + "items": [{ + "input_index": 5, + "sender": "0x3333333333333333333333333333333333333333", + "payload": "0x00ff80", + "block_number": 1230, + "block_timestamp": 1700014760_u64, + "transaction_hash": "0x4444444444444444444444444444444444444444444444444444444444444444" + }] + }) + .to_string() + } + + #[tokio::test] + async fn discovery_decodes_history_and_encodes_optional_era() { + for expected_era in [None, Some(ERA.parse().unwrap())] { + let (client, server) = serve_once(200, history_body()).await; + let info = client.history(expected_era, None).await.unwrap(); + assert_eq!(info.history.available_from.get(), 7); + assert_eq!(info.history.head.get(), 12); + assert_eq!(info.history.version.era_id.to_string(), ERA); + assert_eq!(info.baseline.l1_stop_block, 1240); + assert!(info.compatibility.is_none()); + let url = request_url(&server.await.unwrap()); + assert_eq!(url.path(), "/history"); + let query: Vec<_> = url.query_pairs().collect(); + if expected_era.is_some() { + assert_eq!(query, [("era_id".into(), ERA.into())]); + } else { + assert!(query.is_empty()); + } + } + } + + #[tokio::test] + async fn compatibility_encodes_checkpoint_generation_and_decodes_preserved_boundary() { + let mut body: serde_json::Value = serde_json::from_str(&history_body()).unwrap(); + body["compatibility"] = serde_json::json!({ + "from_generation": 1, + "preserved_input_count": 9 + }); + let (client, server) = serve_once(200, body.to_string()).await; + let info = client + .history(Some(ERA.parse().unwrap()), Some(RecoveryGeneration::new(1))) + .await + .unwrap(); + let compatibility = info.compatibility.unwrap(); + assert_eq!(compatibility.from_generation.get(), 1); + assert_eq!(compatibility.preserved_input_count.get(), 9); + assert_eq!(info.history.version.recovery_generation.get(), 2); + let url = request_url(&server.await.unwrap()); + assert_eq!(url.path(), "/history"); + assert_eq!( + url.query_pairs().collect::>(), + [ + ("era_id".into(), ERA.into()), + ("from_generation".into(), "1".into()) + ] + ); + } + + #[tokio::test] + async fn pages_encode_one_selector_and_decode_original_binary_fields() { + for (start, limit, selector, value) in [ + ( + HistoricalL1InputStart::NextInputIndex(5), + Some(1), + "next_input_index", + "5", + ), + ( + HistoricalL1InputStart::AfterBlock(10), + None, + "after_block", + "10", + ), + ] { + let (client, server) = serve_once(200, page_body()).await; + let page = client + .historical_l1_inputs(ERA.parse().unwrap(), start, limit) + .await + .unwrap(); + assert_eq!(page.next_input_index, 6); + assert_eq!(page.items[0].input_index, 5); + assert_eq!(page.items[0].payload.as_ref(), &[0x00, 0xff, 0x80]); + assert_eq!(page.items[0].sender.as_slice(), &[0x33; 20]); + assert_eq!(page.items[0].transaction_hash.as_slice(), &[0x44; 32]); + let url = request_url(&server.await.unwrap()); + assert_eq!(url.path(), "/historical-l1-inputs"); + let query: std::collections::BTreeMap<_, _> = url.query_pairs().collect(); + assert_eq!(query.get("era_id").unwrap(), ERA); + assert_eq!(query.get(selector).unwrap(), value); + assert_eq!(query.len(), if limit.is_some() { 3 } else { 2 }); + if let Some(limit) = limit { + assert_eq!(query.get("limit").unwrap(), &limit.to_string()); + } + } + } + + #[tokio::test] + async fn era_change_is_typed_for_both_reads() { + let policy = HistoryPolicyError::EraChanged { + current: crate::HistoryVersion { + era_id: "11111111-1111-4111-8111-111111111111".parse().unwrap(), + recovery_generation: sequencer_core::history::RecoveryGeneration::new(0), + }, + }; + for page_request in [false, true] { + let (client, server) = serve_once(409, serde_json::to_string(&policy).unwrap()).await; + let result = if page_request { + client + .historical_l1_inputs( + ERA.parse().unwrap(), + HistoricalL1InputStart::NextInputIndex(0), + None, + ) + .await + .map(|_| ()) + } else { + client + .history(Some(ERA.parse().unwrap()), None) + .await + .map(|_| ()) + }; + assert!(matches!(result, Err(HistoryReadError::History(actual)) if actual == policy)); + server.await.unwrap(); + } + } + + #[tokio::test] + async fn other_refusals_preserve_status_and_body() { + for status in [400, 409, 429, 503] { + let body = format!("refusal {status}"); + let (client, server) = serve_once(status, body.clone()).await; + assert!(matches!( + client.history(None, None).await, + Err(HistoryReadError::Http { status: actual_status, body: actual_body }) + if actual_status == status && actual_body == body + )); + server.await.unwrap(); + } + } + + #[tokio::test] + async fn malformed_success_is_a_decode_error() { + let (client, server) = serve_once(200, "{}".into()).await; + assert!(matches!( + client.history(None, None).await, + Err(HistoryReadError::Decode(_)) + )); + server.await.unwrap(); + } + + #[tokio::test] + async fn history_deadline_covers_the_response_body() { + for page_request in [false, true] { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let endpoint = format!("http://{}", listener.local_addr().unwrap()); + let server = tokio::spawn(async move { + let (mut stream, _) = listener.accept().await.unwrap(); + request_headers(&mut stream).await; + stream + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 100\r\n\r\n{") + .await + .unwrap(); + std::future::pending::<()>().await; + }); + let client = SequencerClient::new(endpoint) + .unwrap() + .with_request_timeout(Duration::from_millis(100)); + let result = tokio::time::timeout(Duration::from_secs(5), async { + if page_request { + client + .historical_l1_inputs( + ERA.parse().unwrap(), + HistoricalL1InputStart::NextInputIndex(0), + None, + ) + .await + .map(|_| ()) + } else { + client.history(None, None).await.map(|_| ()) + } + }) + .await + .expect("the configured deadline must include body transfer"); + server.abort(); + assert!(matches!(result, Err(HistoryReadError::Request(error)) if error.is_timeout())); + } + } +} diff --git a/sdk/rust-client/src/lib.rs b/sdk/rust-client/src/lib.rs index 7003c5b..79302ec 100644 --- a/sdk/rust-client/src/lib.rs +++ b/sdk/rust-client/src/lib.rs @@ -2,13 +2,20 @@ // SPDX-License-Identifier: Apache-2.0 (see LICENSE) mod errors; +mod history; pub use errors::{ - ClientBuildError, GetFeeError, SnapshotError, SubmitRejected, SubmitTxError, SubscribeError, + ClientBuildError, GetFeeError, HistoryReadError, SnapshotError, SubmitRejected, SubmitTxError, + SubscribeError, }; pub use sequencer_core::history::{ - ExecutedInputCount, HistoryClaim, HistoryPolicyError, HistoryVersion, + EraId, ExecutedInputCount, HistoryBounds, HistoryClaim, HistoryPolicyError, HistoryVersion, + RecoveryGeneration, +}; +pub use sequencer_core::history_api::{ + AcceptedCheckpoint, HistoricalL1Input, HistoricalL1InputStart, HistoricalL1InputsPage, + HistoryBaseline, HistoryCompatibility, HistoryDeployment, HistoryInfo, }; use sequencer_core::api::{FeeResponse, TxRequest, TxResponse}; diff --git a/sequencer-core/src/history_api.rs b/sequencer-core/src/history_api.rs new file mode 100644 index 0000000..d8312fb --- /dev/null +++ b/sequencer-core/src/history_api.rs @@ -0,0 +1,81 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +//! Historical scheduler inputs and their handoff to the application feed. + +use alloy_primitives::{Address, B256, Bytes}; +use serde::{Deserialize, Serialize}; + +use crate::history::{EraId, ExecutedInputCount, HistoryBounds, RecoveryGeneration}; + +pub const HISTORICAL_INPUT_MAX_ITEMS: usize = 256; +/// A larger first input is served alone, preserving progress through any history. +pub const HISTORICAL_INPUT_PAYLOAD_TARGET_BYTES: usize = 1024 * 1024; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct HistoryDeployment { + pub chain_id: u64, + pub app_address: Address, + pub input_box_address: Address, + pub app_deployment_block: u64, + pub batch_submitter_address: Address, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub struct HistoryBaseline { + pub l1_stop_block: u64, + /// Exclusive end of the per-application InputBox prefix through the stop block. + pub l1_end_input_index: u64, + pub next_batch_nonce: u64, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub struct AcceptedCheckpoint { + pub inclusion_block: u64, + pub executed_input_count: ExecutedInputCount, + pub next_batch_nonce: u64, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct HistoryInfo { + pub deployment: HistoryDeployment, + pub history: HistoryBounds, + pub baseline: HistoryBaseline, + /// The accepted boundary shares `history.version`; it does not certify client state. + pub accepted_checkpoint: Option, + pub compatibility: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub struct HistoryCompatibility { + pub from_generation: RecoveryGeneration, + /// Checkpoint counts up to and including this boundary preserve their input prefix. + pub preserved_input_count: ExecutedInputCount, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HistoricalL1InputStart { + NextInputIndex(u64), + AfterBlock(u64), +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct HistoricalL1Input { + pub input_index: u64, + pub sender: Address, + /// Original inner application/batch payload, including malformed batches. + pub payload: Bytes, + pub block_number: u64, + pub block_timestamp: u64, + pub transaction_hash: B256, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct HistoricalL1InputsPage { + pub era_id: EraId, + pub l1_stop_block: u64, + pub end_input_index: u64, + /// Only equality with `end_input_index` establishes completion, not page length. + pub next_input_index: u64, + pub items: Vec, +} diff --git a/sequencer-core/src/lib.rs b/sequencer-core/src/lib.rs index 7c52133..d8ab1f3 100644 --- a/sequencer-core/src/lib.rs +++ b/sequencer-core/src/lib.rs @@ -10,6 +10,7 @@ pub mod batch; pub mod broadcast; pub mod fee; pub mod history; +pub mod history_api; pub mod l2_tx; pub mod protocol; pub mod scheduler; diff --git a/sequencer/src/egress/api/history.rs b/sequencer/src/egress/api/history.rs new file mode 100644 index 0000000..e14cd26 --- /dev/null +++ b/sequencer/src/egress/api/history.rs @@ -0,0 +1,276 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +//! Coherent history metadata and bounded pages of the immutable era L1 prefix. + +use std::io::Cursor; +use std::pin::Pin; +use std::sync::Arc; +use std::task::{Context, Poll}; + +use axum::body::Body; +use axum::extract::rejection::QueryRejection; +use axum::extract::{Query, State}; +use axum::http::{StatusCode, header}; +use axum::response::{IntoResponse, Response}; +use axum::routing::get; +use axum::{Json, Router}; +use sequencer_core::history::{EraId, RecoveryGeneration}; +use sequencer_core::history_api::{HISTORICAL_INPUT_MAX_ITEMS, HistoricalL1InputStart}; +use serde::Deserialize; +use tokio::io::{AsyncRead, ReadBuf}; +use tokio::sync::{OwnedSemaphorePermit, Semaphore}; +use tokio_util::io::ReaderStream; + +use crate::http::{ApiError, StorageTaskError, storage_task}; +use crate::runtime::shutdown::RuntimeScope; +use crate::storage::{HistoricalReadError, Storage}; + +const MAX_HISTORICAL_RESPONSES: usize = 8; + +struct HistoryState { + db_path: String, + shutdown: RuntimeScope, + responses: Arc, +} + +pub(super) fn router(db_path: String, shutdown: RuntimeScope) -> Router { + Router::new() + .route("/history", get(history)) + .route("/historical-l1-inputs", get(historical_l1_inputs)) + .with_state(Arc::new(HistoryState { + db_path, + shutdown, + responses: Arc::new(Semaphore::new(MAX_HISTORICAL_RESPONSES)), + })) +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct HistoryQuery { + era_id: Option, + from_generation: Option, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct HistoricalInputsQuery { + era_id: EraId, + next_input_index: Option, + after_block: Option, + limit: Option, +} + +impl HistoricalInputsQuery { + fn start(&self) -> Result { + match (self.next_input_index, self.after_block) { + (Some(index), None) => Ok(HistoricalL1InputStart::NextInputIndex(index)), + (None, Some(block)) => Ok(HistoricalL1InputStart::AfterBlock(block)), + _ => Err(ApiError::bad_request( + "provide exactly one of next_input_index or after_block", + )), + } + } +} + +async fn history( + State(state): State>, + query: Result, QueryRejection>, +) -> Response { + let Query(query) = match query { + Ok(query) => query, + Err(error) => return ApiError::bad_request(error.body_text()).into_response(), + }; + if query.from_generation.is_some() && query.era_id.is_none() { + return ApiError::bad_request("from_generation requires era_id").into_response(); + } + if state.shutdown.is_shutdown_requested() { + return ApiError::unavailable("sequencer shutting down").into_response(); + } + let db_path = state.db_path.clone(); + match storage_task(state.shutdown.clone(), "read history metadata", move |_| { + Ok(Storage::open_read_only(&db_path)?.history_info(query.era_id, query.from_generation)?) + }) + .await + { + Ok(info) => Json(info).into_response(), + Err(error) => read_error(error), + } +} + +async fn historical_l1_inputs( + State(state): State>, + query: Result, QueryRejection>, +) -> Response { + let Query(query) = match query { + Ok(query) => query, + Err(error) => return ApiError::bad_request(error.body_text()).into_response(), + }; + let start = match query.start() { + Ok(start) => start, + Err(error) => return error.into_response(), + }; + if state.shutdown.is_shutdown_requested() { + return ApiError::unavailable("sequencer shutting down").into_response(); + } + let permit = match state.responses.clone().try_acquire_owned() { + Ok(permit) => permit, + Err(_) => return ApiError::overloaded("historical response limit reached").into_response(), + }; + let db_path = state.db_path.clone(); + let result = storage_task( + state.shutdown.clone(), + "read historical L1 inputs", + move |_| { + // The blocking task owns admission even if its HTTP request is cancelled. + let page = Storage::open_read_only(&db_path)?.historical_l1_inputs( + query.era_id, + start, + query.limit.unwrap_or(HISTORICAL_INPUT_MAX_ITEMS), + )?; + let bytes = serde_json::to_vec(&page)?; + Ok((bytes, permit)) + }, + ) + .await; + match result { + Ok((bytes, permit)) => ( + [(header::CONTENT_TYPE, "application/json")], + Body::from_stream(ReaderStream::new(HistoricalResponseBody { + bytes: Cursor::new(bytes), + _permit: permit, + })), + ) + .into_response(), + Err(error) => read_error(error), + } +} + +fn read_error(error: StorageTaskError) -> Response { + match error.downcast_ref::() { + Some(HistoricalReadError::Policy(policy)) => { + return (StatusCode::CONFLICT, Json(*policy)).into_response(); + } + Some(HistoricalReadError::BadRequest(message)) => { + return ApiError::bad_request(message.clone()).into_response(); + } + _ => {} + } + tracing::warn!(%error, "history read unavailable"); + ApiError::unavailable("history read unavailable").into_response() +} + +struct HistoricalResponseBody { + bytes: Cursor>, + _permit: OwnedSemaphorePermit, +} + +impl AsyncRead for HistoricalResponseBody { + fn poll_read( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &mut ReadBuf<'_>, + ) -> Poll> { + Pin::new(&mut self.bytes).poll_read(cx, buf) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const ERA: &str = "11111111-1111-4111-8111-111111111111"; + + fn state() -> Arc { + Arc::new(HistoryState { + db_path: "unused: invalid queries do not access storage".to_owned(), + shutdown: RuntimeScope::default(), + responses: Arc::new(Semaphore::new(1)), + }) + } + + #[tokio::test] + async fn malformed_queries_return_bad_request_json_before_storage() { + for query in [ + "", + "?era_id=bad&next_input_index=0", + &format!("?era_id={ERA}"), + &format!("?era_id={ERA}&next_input_index=0&after_block=0"), + &format!("?era_id={ERA}&next_input_index=-1"), + &format!("?era_id={ERA}&next_input_index=0&unrecognized=1"), + ] { + let uri = format!("/historical-l1-inputs{query}").parse().unwrap(); + let response = historical_l1_inputs(State(state()), Query::try_from_uri(&uri)).await; + assert_eq!(response.status(), StatusCode::BAD_REQUEST, "{query}"); + let bytes = axum::body::to_bytes(response.into_body(), 4096) + .await + .unwrap(); + let error: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + assert_eq!(error["code"], "BAD_REQUEST"); + } + for query in [ + "?from_generation=0", + &format!("?era_id={ERA}&from_generation=-1"), + &format!("?era_id={ERA}&from_generation=invalid"), + &format!("?era_id={ERA}&unrecognized=1"), + ] { + let uri = format!("/history{query}").parse().unwrap(); + let response = history(State(state()), Query::try_from_uri(&uri)).await; + assert_eq!(response.status(), StatusCode::BAD_REQUEST, "{query}"); + let bytes = axum::body::to_bytes(response.into_body(), 4096) + .await + .unwrap(); + let error: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + assert_eq!(error["code"], "BAD_REQUEST"); + } + } + + #[tokio::test] + async fn shutdown_and_overload_refuse_before_opening_storage() { + let state = state(); + let uri = format!("/historical-l1-inputs?era_id={ERA}&next_input_index=0") + .parse() + .unwrap(); + let permit = state.responses.clone().try_acquire_owned().unwrap(); + let response = historical_l1_inputs(State(state.clone()), Query::try_from_uri(&uri)).await; + assert_eq!(response.status(), StatusCode::TOO_MANY_REQUESTS); + drop(permit); + state.shutdown.request_shutdown(); + let response = historical_l1_inputs(State(state.clone()), Query::try_from_uri(&uri)).await; + assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); + for query in [ + "".to_owned(), + format!("?era_id={ERA}"), + format!("?era_id={ERA}&from_generation=0"), + ] { + let uri = format!("/history{query}").parse().unwrap(); + let response = history(State(state.clone()), Query::try_from_uri(&uri)).await; + assert_eq!( + response.status(), + StatusCode::SERVICE_UNAVAILABLE, + "{query}" + ); + } + } + + #[tokio::test] + async fn response_body_holds_admission_until_consumed_or_dropped() { + let permits = Arc::new(Semaphore::new(1)); + for consume in [false, true] { + let body = Body::from_stream(ReaderStream::new(HistoricalResponseBody { + bytes: Cursor::new(vec![1; 100_000]), + _permit: permits.clone().try_acquire_owned().unwrap(), + })); + assert!(permits.clone().try_acquire_owned().is_err()); + if consume { + assert_eq!( + axum::body::to_bytes(body, 100_000).await.unwrap().len(), + 100_000 + ); + } else { + drop(body); + } + assert_eq!(permits.available_permits(), 1); + } + } +} diff --git a/sequencer/src/egress/api/mod.rs b/sequencer/src/egress/api/mod.rs index 39a7db9..37e2d64 100644 --- a/sequencer/src/egress/api/mod.rs +++ b/sequencer/src/egress/api/mod.rs @@ -1,10 +1,10 @@ // (c) Cartesi and individual authors (see AUTHORS) // SPDX-License-Identifier: Apache-2.0 (see LICENSE) -//! Egress HTTP API routes: WebSocket subscribe + k8s-style health probes. -//! Additional read endpoints will land here. +//! Internal history, snapshot, WebSocket, and health endpoints. mod health; +mod history; mod snapshot; mod state; mod subscribe; @@ -31,6 +31,7 @@ pub(crate) fn router( shutdown: RuntimeScope, snapshot_release_scheduler: ReleaseScheduler, ) -> Router { + let history_router = history::router(snapshot_state.db_path.clone(), shutdown.clone()); let subscribe_router = Router::new() .route("/ws/subscribe", get(subscribe::subscribe_l2_txs)) .with_state(subscribe_state); @@ -43,6 +44,7 @@ pub(crate) fn router( subscribe_router .merge(health_router) + .merge(history_router) .merge(snapshot::router( snapshot_state, shutdown, diff --git a/sequencer/src/integration_tests/historical_bootstrap.rs b/sequencer/src/integration_tests/historical_bootstrap.rs new file mode 100644 index 0000000..bfffbd4 --- /dev/null +++ b/sequencer/src/integration_tests/historical_bootstrap.rs @@ -0,0 +1,553 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +//! A reader restores its own transfer history, replays raw L1 through the +//! canonical scheduler, and crosses the rebuilt baseline into the live feed. + +mod recovery_compatibility; + +use std::net::SocketAddr; +use std::time::Duration; + +use alloy::signers::{SignerSync, local::PrivateKeySigner}; +use alloy_primitives::{Address, B256, U256}; +use alloy_sol_types::{Eip712Domain, SolCall, SolStruct}; +use app_core::application::{ + DepositNotice, Method, Transfer, TransferNotice, WalletApp, WalletConfig, +}; +use futures_util::StreamExt; +use sequencer_core::api::WsTxMessage; +use sequencer_core::application::{ + AppOutput, AppOutputs, Application, CanonicalState, execute_direct_input, +}; +use sequencer_core::batch::{Batch, Frame, WireUserOp}; +use sequencer_core::history::{ExecutedInputCount, HistoryClaim, HistoryPolicyError}; +use sequencer_core::history_api::{AcceptedCheckpoint, HistoricalL1InputStart}; +use sequencer_core::l2_tx::DirectInput; +use sequencer_core::scheduler::{ + BatchRejectReason, ProcessOutcome, Scheduler, SchedulerConfig, SchedulerInput, +}; +use sequencer_core::user_op::UserOp; +use sequencer_rust_client::{HistoryReadError, SequencerClient}; +use ssz::Encode; +use tokio::sync::mpsc; +use tokio_tungstenite::tungstenite::Message; + +use crate::egress::l2_tx_feed::{L2TxFeed, L2TxFeedConfig}; +use crate::http::{self, ApiConfig}; +use crate::ingress::inclusion_lane::{PendingUserOp, dump_info}; +use crate::runtime::shutdown::RuntimeScope; +use crate::storage::test_helpers::{default_protocol_timing, pin_test_deployment_identity}; +use crate::storage::{ + DirectInputExecution, FrontierMode, IngestedSafeInput, LifecycleCommand, SafeInputRange, + Storage, +}; + +use super::common::temp_db; + +const SUBMITTER: Address = Address::repeat_byte(0x33); +const APP: Address = Address::repeat_byte(0x11); +const RECIPIENT: Address = Address::repeat_byte(0x44); +const STOP: u64 = 1240; + +fn domain() -> Eip712Domain { + sequencer_core::build_input_domain(1, APP) +} + +fn config() -> WalletConfig { + WalletConfig { + sequencer_address: SUBMITTER, + ..WalletConfig::default() + } +} + +fn deposit(block: u64, recipient: Address, amount: u64) -> IngestedSafeInput { + let mut payload = Vec::new(); + payload.extend_from_slice(config().supported_erc20_token.as_slice()); + payload.extend_from_slice(recipient.as_slice()); + payload.extend_from_slice(&U256::from(amount).to_be_bytes::<32>()); + raw(config().erc20_portal_address, block, payload) +} + +fn raw(sender: Address, block: u64, payload: Vec) -> IngestedSafeInput { + IngestedSafeInput { + sender, + payload, + block_number: block, + block_timestamp: 1_700_000_000 + block * 12, + transaction_hash: B256::repeat_byte(u8::try_from(block % 251).unwrap()), + } +} + +fn transfer_batch( + signer: &PrivateKeySigner, + nonce: u32, + block: u64, + safe_block: u64, + amount: u64, +) -> IngestedSafeInput { + let op = UserOp { + nonce, + max_fee: 0, + data: Method::Transfer(Transfer { + amount: U256::from(amount), + to: RECIPIENT, + }) + .as_ssz_bytes() + .into(), + }; + let signature = signer + .sign_hash_sync(&op.eip712_signing_hash(&domain())) + .unwrap(); + raw( + SUBMITTER, + block, + Batch { + nonce: u64::from(nonce), + frames: vec![Frame { + safe_block, + fee_price: 0, + user_ops: vec![WireUserOp { + nonce, + max_fee: op.max_fee, + data: op.data.to_vec(), + signature: signature.as_bytes().to_vec(), + }], + }], + } + .as_ssz_bytes(), + ) +} + +fn notices(outputs: AppOutputs) -> Vec> { + outputs + .into_iter() + .map(|output| match output { + AppOutput::Notice(bytes) => bytes, + other => panic!("unexpected output in transfer-history fixture: {other:?}"), + }) + .collect() +} + +fn execute( + scheduler: &mut Scheduler, + input: &IngestedSafeInput, +) -> sequencer_core::scheduler::ProcessResult { + scheduler + .process_input(SchedulerInput { + sender: input.sender, + inclusion_block: input.block_number, + domain: domain(), + payload: input.payload.clone(), + }) + .unwrap() +} + +struct Server { + addr: SocketAddr, + shutdown: RuntimeScope, + task: Option, + _rx: mpsc::Receiver, +} + +impl Drop for Server { + fn drop(&mut self) { + self.shutdown.request_shutdown(); + if let Some(task) = self.task.take() { + task.abort(); + } + } +} + +impl Server { + async fn start(db_path: &str) -> Self { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let shutdown = RuntimeScope::default(); + let (tx, rx) = mpsc::channel(1); + let feed = L2TxFeed::new( + db_path.to_owned(), + shutdown.clone(), + L2TxFeedConfig::default(), + ); + let task = http::start_on_listener( + listener, + tx, + shutdown.clone(), + feed, + ApiConfig::new(domain(), WalletApp::max_method_payload_bytes()), + http::SnapshotState { + db_path: db_path.to_owned(), + state_file_in_dump: |prefix| { + WalletApp::state_file_in_dump(&dump_info::app_prefix(prefix)) + }, + }, + ); + Self { + addr, + shutdown, + task: Some(task), + _rx: rx, + } + } + + async fn stop(mut self) { + self.shutdown.request_shutdown(); + tokio::time::timeout(Duration::from_secs(3), self.task.take().unwrap()) + .await + .unwrap() + .unwrap() + .unwrap(); + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn historical_bootstrap_restores_transfer_history_and_hands_off_at_baseline() { + let signer: PrivateKeySigner = format!("{:064x}", 1).parse().unwrap(); + let user = signer.address(); + let inputs = vec![ + deposit(5, user, 100), + deposit(12, user, 20), + transfer_batch(&signer, 0, 20, 10, 7), + deposit(20, user, 30), + deposit(24, user, 40), + raw(SUBMITTER, 1230, vec![0]), + transfer_batch(&signer, 1, 1232, 1228, 11), + deposit(1235, user, 50), + ]; + + let mut reference = Scheduler::new(WalletApp::new(config()), SchedulerConfig::new(SUBMITTER)); + let mut expected_history = Vec::new(); + for input in &inputs { + expected_history.extend(notices(execute(&mut reference, input).outputs)); + } + expected_history.extend(notices(reference.drain_covered_at(STOP).unwrap())); + let (mut reference_app, reference_nonce) = reference.finish(); + assert_eq!(reference_app.executed_input_count().get(), 7); + assert_eq!(reference_app.last_executed_safe_block(), 1235); + assert_eq!(reference_nonce, 2); + let deposit_notice = |amount| { + DepositNotice { + token: config().supported_erc20_token, + sender: user, + amount: U256::from(amount), + } + .abi_encode() + }; + let transfer_notice = |amount| { + TransferNotice { + sender: user, + recipient: RECIPIENT, + amount: U256::from(amount), + } + .abi_encode() + }; + assert_eq!( + expected_history, + vec![ + deposit_notice(100_u64), + transfer_notice(7_u64), + deposit_notice(20), + deposit_notice(30), + deposit_notice(40), + transfer_notice(11), + deposit_notice(50) + ] + ); + + let db = temp_db("historical-projection-bootstrap"); + let dumps = tempfile::tempdir().unwrap(); + let baseline_dump = dumps.path().join("baseline"); + dump_info::create_dump_dir_with_info( + &mut reference_app, + &baseline_dump, + &dump_info::DumpInfo { + format_version: dump_info::FORMAT_VERSION, + next_batch_nonce: reference_nonce, + }, + ) + .unwrap(); + let mut storage = Storage::initialize_for_command(&db.path, LifecycleCommand::Rebuild).unwrap(); + pin_test_deployment_identity(&mut storage, SUBMITTER); + storage + .append_ingested_safe_inputs_with_timestamp( + STOP, + 1_700_000_000 + STOP * 12, + &inputs, + SUBMITTER, + &default_protocol_timing(), + FrontierMode::DeferUntilAnchorSet, + ) + .unwrap(); + storage + .complete_baseline_setup( + &baseline_dump, + reference_app.executed_input_count(), + STOP, + reference_nonce, + true, + ) + .unwrap(); + let server = Server::start(&db.path).await; + let client = SequencerClient::new(format!("http://{}", server.addr)).unwrap(); + let metadata = client.history(None, None).await.unwrap(); + let era = metadata.history.version.era_id; + assert_eq!(metadata.history.available_from.get(), 7); + assert_eq!(metadata.history.head.get(), 7); + assert_eq!(metadata.baseline.l1_stop_block, STOP); + assert_eq!(metadata.baseline.l1_end_input_index, 8); + assert_eq!(metadata.baseline.next_batch_nonce, reference_nonce); + assert_eq!(metadata.accepted_checkpoint, None); + assert_eq!(metadata.deployment.batch_submitter_address, SUBMITTER); + assert_eq!( + sequencer_core::build_input_domain( + metadata.deployment.chain_id, + metadata.deployment.app_address + ), + domain() + ); + let mut other_era_bytes = *era.as_bytes(); + other_era_bytes[0] ^= 1; + let other_era = sequencer_core::history::EraId::from_bytes(other_era_bytes).unwrap(); + assert!(matches!( + client.history(Some(other_era), None).await, + Err(HistoryReadError::History(HistoryPolicyError::EraChanged { current })) + if current == metadata.history.version + )); + assert!(matches!( + client + .historical_l1_inputs( + other_era, + HistoricalL1InputStart::NextInputIndex(u64::MAX), + None + ) + .await, + Err(HistoryReadError::History( + HistoryPolicyError::EraChanged { .. } + )) + )); + assert!(matches!( + client + .historical_l1_inputs(era, HistoricalL1InputStart::NextInputIndex(9), None) + .await, + Err(HistoryReadError::Http { status: 400, .. }) + )); + + // The reader's saved projection includes notices that the core wallet dump + // cannot reconstruct. Prepare a trusted end-of-block B=20 checkpoint. + let mut checkpoint_scheduler = + Scheduler::new(WalletApp::new(config()), SchedulerConfig::new(SUBMITTER)); + let mut saved_history = Vec::new(); + for input in &inputs[..4] { + saved_history.extend(notices(execute(&mut checkpoint_scheduler, input).outputs)); + } + assert_eq!(checkpoint_scheduler.queued_direct_len(), 2); + let (mut checkpoint_app, next_batch_nonce) = checkpoint_scheduler.finish(); + let receipt = AcceptedCheckpoint { + inclusion_block: 20, + executed_input_count: checkpoint_app.executed_input_count(), + next_batch_nonce, + }; + assert_eq!(receipt.executed_input_count.get(), 2); + assert_eq!(checkpoint_app.last_executed_safe_block(), 10); + let backup = dumps.path().join("reader-checkpoint"); + std::fs::create_dir(&backup).unwrap(); + checkpoint_app.create_dump(&backup.join("wallet")).unwrap(); + std::fs::write( + backup.join("projection.json"), + serde_json::to_vec(&(receipt, &saved_history)).unwrap(), + ) + .unwrap(); + drop(checkpoint_app); + drop(saved_history); + + let restored_app = WalletApp::from_dump(&backup.join("wallet")).unwrap(); + let (receipt, mut history): (AcceptedCheckpoint, Vec>) = + serde_json::from_slice(&std::fs::read(backup.join("projection.json")).unwrap()).unwrap(); + assert_eq!( + restored_app.executed_input_count(), + receipt.executed_input_count + ); + let mut start = HistoricalL1InputStart::AfterBlock(restored_app.last_executed_safe_block()); + let mut replay = Scheduler::resume_at( + restored_app, + SchedulerConfig::new(SUBMITTER), + receipt.next_batch_nonce, + ); + std::fs::remove_dir_all(backup).unwrap(); + + let mut seen = Vec::new(); + loop { + // One row per page forces a page split between the batch at B and the + // direct arriving later in that same block. + let page = client + .historical_l1_inputs(era, start, Some(1)) + .await + .unwrap(); + assert_eq!(page.era_id, era); + assert_eq!(page.l1_stop_block, STOP); + assert_eq!(page.end_input_index, 8); + for item in page.items { + let source = &inputs[usize::try_from(item.input_index).unwrap()]; + assert_eq!(item.payload.as_ref(), source.payload); + assert_eq!(item.sender, source.sender); + assert_eq!(item.block_timestamp, source.block_timestamp); + assert_eq!(item.transaction_hash, source.transaction_hash); + seen.push(item.input_index); + if item.block_number <= receipt.inclusion_block { + if item.sender != metadata.deployment.batch_submitter_address { + replay.enqueue_direct(item.sender, item.block_number, item.payload.to_vec()); + } + } else { + let result = replay + .process_input(SchedulerInput { + sender: item.sender, + inclusion_block: item.block_number, + domain: domain(), + payload: item.payload.to_vec(), + }) + .unwrap(); + if item.input_index == 5 { + assert_eq!( + result.outcome, + ProcessOutcome::BatchRejected(BatchRejectReason::DecodeFailed) + ); + assert_eq!( + result.outputs.len(), + 3, + "malformed batch still drains overdue directs" + ); + assert_eq!(replay.next_expected_batch_nonce(), 1); + } + history.extend(notices(result.outputs)); + } + } + if page.next_input_index == page.end_input_index { + break; + } + start = HistoricalL1InputStart::NextInputIndex(page.next_input_index); + } + assert_eq!(seen, vec![1, 2, 3, 4, 5, 6, 7]); + assert_eq!( + history.len(), + 6, + "the young final direct waits until raw EOF" + ); + assert_eq!(replay.queued_direct_len(), 1); + history.extend(notices( + replay + .drain_covered_at(metadata.baseline.l1_stop_block) + .unwrap(), + )); + let (mut reader_app, nonce) = replay.finish(); + assert_eq!(nonce, metadata.baseline.next_batch_nonce); + assert_eq!( + reader_app.executed_input_count(), + metadata.history.available_from + ); + assert_eq!( + reader_app.canonical_snapshot_bytes().unwrap(), + reference_app.canonical_snapshot_bytes().unwrap() + ); + assert_eq!(history, expected_history); + let eof = client + .historical_l1_inputs(era, HistoricalL1InputStart::NextInputIndex(8), None) + .await + .unwrap(); + assert!(eof.items.is_empty()); + assert_eq!(eof.next_input_index, eof.end_input_index); + + let current = client.history(Some(era), None).await.unwrap(); + assert_eq!(current.baseline, metadata.baseline); + let mut stream = client + .subscribe(HistoryClaim { + version: current.history.version, + next_input: reader_app.executed_input_count(), + }) + .await + .unwrap(); + let live = deposit(1245, user, 60); + storage + .append_ingested_safe_inputs_with_timestamp( + 1245, + live.block_timestamp, + std::slice::from_ref(&live), + SUBMITTER, + &default_protocol_timing(), + FrontierMode::Populate, + ) + .unwrap(); + let mut head = storage.open_state().unwrap().unwrap(); + let execution = execute_direct_input( + &mut reference_app, + &DirectInput { + sender: live.sender, + block_number: live.block_number, + payload: live.payload.clone(), + }, + ) + .unwrap(); + expected_history.extend(notices(execution.outputs)); + storage + .close_frame_only_with_executions( + &mut head, + 1245, + SafeInputRange::new(8, 9), + &[DirectInputExecution { + safe_input_index: 8, + executed_input_offset: execution.offset, + }], + ) + .unwrap(); + let message = tokio::time::timeout(Duration::from_secs(3), stream.next()) + .await + .unwrap() + .unwrap() + .unwrap(); + let Message::Text(text) = message else { + panic!("expected application input") + }; + let message: WsTxMessage = serde_json::from_str(text.as_str()).unwrap(); + let WsTxMessage::DirectInput { + offset, + sender, + block_number, + payload, + input_index, + batch_nonce, + .. + } = message + else { + panic!("expected post-baseline deposit") + }; + assert_eq!(offset, 7, "terminal-drained D4 is not delivered again"); + assert_eq!(input_index, 8); + assert_eq!(batch_nonce, 2); + let execution = execute_direct_input( + &mut reader_app, + &DirectInput { + sender: sender.parse().unwrap(), + block_number, + payload: alloy_primitives::hex::decode(payload).unwrap(), + }, + ) + .unwrap(); + assert_eq!(execution.offset, ExecutedInputCount::new(7)); + history.extend(notices(execution.outputs)); + assert_eq!(history, expected_history); + assert_eq!( + reader_app.canonical_snapshot_bytes().unwrap(), + reference_app.canonical_snapshot_bytes().unwrap() + ); + let fixed = client + .historical_l1_inputs(era, HistoricalL1InputStart::AfterBlock(STOP), None) + .await + .unwrap(); + assert!( + fixed.items.is_empty(), + "new L1 inputs never extend this era's historical prefix" + ); + assert_eq!(fixed.end_input_index, 8); + drop(stream); + server.stop().await; +} diff --git a/sequencer/src/integration_tests/historical_bootstrap/recovery_compatibility.rs b/sequencer/src/integration_tests/historical_bootstrap/recovery_compatibility.rs new file mode 100644 index 0000000..165ed23 --- /dev/null +++ b/sequencer/src/integration_tests/historical_bootstrap/recovery_compatibility.rs @@ -0,0 +1,403 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +use std::path::{Path, PathBuf}; +use std::time::SystemTime; + +use sequencer_core::application::{ExecutionOutcome, validate_and_execute_user_op}; +use sequencer_core::history::{HistoryVersion, RecoveryGeneration}; +use sequencer_core::user_op::SignedUserOp; +use sequencer_rust_client::{ + HistoryPolicyError, HistoryReadError, SubscribeError, SubscribeStream, +}; +use tokio::sync::oneshot; + +use super::*; +use crate::ingress::inclusion_lane::IncludedUserOp; +use crate::storage::WriteHead; +use crate::storage::test_helpers::local_batch_payload; + +struct Backup { + claim: HistoryClaim, + path: PathBuf, +} + +impl Backup { + fn save( + directory: &Path, + name: &str, + app: &mut WalletApp, + history: &[Vec], + version: HistoryVersion, + ) -> Self { + let claim = HistoryClaim { + version, + next_input: app.executed_input_count(), + }; + let path = directory.join(name); + std::fs::create_dir(&path).unwrap(); + app.create_dump(&path.join("wallet")).unwrap(); + std::fs::write( + path.join("projection.json"), + serde_json::to_vec(&(claim, history)).unwrap(), + ) + .unwrap(); + Self { claim, path } + } + + fn restore(&self) -> (WalletApp, Vec>) { + let app = WalletApp::from_dump(&self.path.join("wallet")).unwrap(); + let (claim, history): (HistoryClaim, Vec>) = + serde_json::from_slice(&std::fs::read(self.path.join("projection.json")).unwrap()) + .unwrap(); + assert_eq!(claim, self.claim); + assert_eq!(app.executed_input_count(), claim.next_input); + (app, history) + } +} + +fn append_transfer( + storage: &mut Storage, + head: &mut WriteHead, + app: &mut WalletApp, + history: &mut Vec>, + signer: &PrivateKeySigner, + amount: u64, +) { + let op = UserOp { + nonce: app.current_user_nonce(signer.address()), + max_fee: head.frame_fee, + data: Method::Transfer(Transfer { + to: RECIPIENT, + amount: U256::from(amount), + }) + .as_ssz_bytes() + .into(), + }; + let outcome = + validate_and_execute_user_op(app, signer.address(), &op, head.frame_fee, head.safe_block) + .unwrap(); + let ExecutionOutcome::Included(execution) = outcome else { + panic!("fixture transfer rejected: {outcome:?}") + }; + history.extend(notices(execution.outputs)); + let (respond_to, _response) = oneshot::channel(); + let included = IncludedUserOp { + pending: PendingUserOp { + signed: SignedUserOp { + sender: signer.address(), + signature: signer + .sign_hash_sync(&op.eip712_signing_hash(&domain())) + .unwrap(), + user_op: op, + }, + respond_to, + received_at: SystemTime::now(), + }, + executed_input_offset: execution.offset, + }; + storage + .append_executed_user_ops_chunk(head, &[included]) + .unwrap(); +} + +fn close_and_accept( + storage: &mut Storage, + head: &mut WriteHead, + app: &mut WalletApp, + dumps: &Path, + inclusion_block: u64, +) { + let index = head.batch_index; + let nonce = storage.batch_nonce(index).unwrap(); + let prefix = dumps.join(format!("accepted-{index}")); + dump_info::create_dump_dir_with_info( + app, + &prefix, + &dump_info::DumpInfo { + format_version: dump_info::FORMAT_VERSION, + next_batch_nonce: nonce + 1, + }, + ) + .unwrap(); + storage + .close_frame_and_batch_with_snapshot( + head, + head.safe_block, + &prefix, + index, + app.executed_input_count(), + ) + .unwrap(); + let payload = local_batch_payload(storage, nonce); + storage + .append_safe_inputs( + inclusion_block, + &[crate::storage::StoredSafeInput { + sender: SUBMITTER, + block_number: inclusion_block, + payload, + }], + SUBMITTER, + &default_protocol_timing(), + ) + .unwrap(); +} + +fn recover_tip(storage: &mut Storage, head: &mut WriteHead, safe_block: u64) { + let protocol = default_protocol_timing(); + storage + .append_safe_inputs(safe_block, &[], SUBMITTER, &protocol) + .unwrap(); + let invalidated = storage + .recover_aging_tip_for_recovery(head.batch_index, &protocol, crate::clock::unix_now_ms()) + .unwrap(); + assert_eq!(invalidated, vec![head.batch_index]); + *head = storage.open_state().unwrap().unwrap(); +} + +async fn replay_one(stream: &mut SubscribeStream, app: &mut WalletApp, history: &mut Vec>) { + let message = tokio::time::timeout(Duration::from_secs(3), stream.next()) + .await + .unwrap() + .unwrap() + .unwrap(); + let Message::Text(text) = message else { + panic!("expected application input") + }; + let message: WsTxMessage = serde_json::from_str(text.as_str()).unwrap(); + let WsTxMessage::UserOp { + offset, + sender, + nonce, + fee, + data, + safe_block, + .. + } = message + else { + panic!("expected transfer") + }; + assert_eq!(offset, app.executed_input_count().get()); + let op = UserOp { + nonce, + max_fee: fee, + data: alloy_primitives::hex::decode(data).unwrap().into(), + }; + let outcome = + validate_and_execute_user_op(app, sender.parse().unwrap(), &op, fee, safe_block).unwrap(); + let ExecutionOutcome::Included(execution) = outcome else { + panic!("feed transfer rejected: {outcome:?}") + }; + history.extend(notices(execution.outputs)); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn checkpoint_compatibility_survives_missed_recoveries_and_retries_admission_races() { + let db = temp_db("reader-checkpoint-compatibility"); + let dumps = tempfile::tempdir().unwrap(); + let signer: PrivateKeySigner = format!("{:064x}", 1).parse().unwrap(); + let mut app = WalletApp::new(config()); + let mut history = Vec::new(); + let baseline = dumps.path().join("genesis"); + dump_info::create_dump_dir_with_info( + &mut app, + &baseline, + &dump_info::DumpInfo { + format_version: dump_info::FORMAT_VERSION, + next_batch_nonce: 0, + }, + ) + .unwrap(); + let mut storage = Storage::initialize_for_command(&db.path, LifecycleCommand::Setup).unwrap(); + pin_test_deployment_identity(&mut storage, SUBMITTER); + storage + .complete_baseline_setup(&baseline, ExecutedInputCount::ZERO, 0, 0, false) + .unwrap(); + let mut head = storage + .initialize_open_state(0, SafeInputRange::empty_at(0)) + .unwrap(); + let directs = [ + deposit(5, signer.address(), 1_000_000_000_000_000), + deposit(6, RECIPIENT, 100), + ]; + storage + .append_ingested_safe_inputs_with_timestamp( + 10, + crate::clock::unix_now_ms() / 1000, + &directs, + SUBMITTER, + &default_protocol_timing(), + FrontierMode::Populate, + ) + .unwrap(); + let mut executions = Vec::new(); + for (index, direct) in directs.into_iter().enumerate() { + let execution = execute_direct_input( + &mut app, + &DirectInput { + sender: direct.sender, + payload: direct.payload, + block_number: direct.block_number, + }, + ) + .unwrap(); + history.extend(notices(execution.outputs)); + executions.push(DirectInputExecution { + safe_input_index: index as u64, + executed_input_offset: execution.offset, + }); + } + storage + .close_frame_only_with_executions(&mut head, 10, SafeInputRange::new(0, 2), &executions) + .unwrap(); + append_transfer(&mut storage, &mut head, &mut app, &mut history, &signer, 10); + close_and_accept(&mut storage, &mut head, &mut app, dumps.path(), 20); + let generation_zero = storage.history_state().unwrap().version; + let good_zero = Backup::save(dumps.path(), "g0-at3", &mut app, &history, generation_zero); + append_transfer(&mut storage, &mut head, &mut app, &mut history, &signer, 11); + let bad_zero = Backup::save(dumps.path(), "g0-at4", &mut app, &history, generation_zero); + + let server = Server::start(&db.path).await; + let client = SequencerClient::new(format!("http://{}", server.addr)).unwrap(); + let era = generation_zero.era_id; + recover_tip(&mut storage, &mut head, 1500); + (app, history) = good_zero.restore(); + let generation_one = storage.history_state().unwrap().version; + assert_eq!(generation_one.recovery_generation.get(), 1); + append_transfer(&mut storage, &mut head, &mut app, &mut history, &signer, 20); + let middle_one = Backup::save(dumps.path(), "g1-at4", &mut app, &history, generation_one); + append_transfer(&mut storage, &mut head, &mut app, &mut history, &signer, 21); + close_and_accept(&mut storage, &mut head, &mut app, dumps.path(), 1501); + let good_one = Backup::save(dumps.path(), "g1-at5", &mut app, &history, generation_one); + append_transfer(&mut storage, &mut head, &mut app, &mut history, &signer, 22); + let bad_one = Backup::save(dumps.path(), "g1-at6", &mut app, &history, generation_one); + + recover_tip(&mut storage, &mut head, 3000); + (app, history) = good_one.restore(); + for amount in [30, 31, 32] { + append_transfer( + &mut storage, + &mut head, + &mut app, + &mut history, + &signer, + amount, + ); + } + let current = client.history(Some(era), None).await.unwrap(); + assert_eq!(current.history.head.get(), 8); + assert_eq!(current.history.version.recovery_generation.get(), 2); + assert_eq!(current.compatibility, None); + let same = client + .history(Some(era), Some(RecoveryGeneration::new(2))) + .await + .unwrap(); + assert_eq!(same.compatibility.unwrap().preserved_input_count.get(), 8); + let future = Some(RecoveryGeneration::new(u64::MAX)); + assert!(matches!( + client.history(Some(era), future).await, + Err(HistoryReadError::Http { status: 400, .. }) + )); + let mut other_era_bytes = *era.as_bytes(); + other_era_bytes[0] ^= 1; + let other_era = sequencer_core::history::EraId::from_bytes(other_era_bytes).unwrap(); + assert!(matches!( + client.history(Some(other_era), future).await, + Err(HistoryReadError::History(HistoryPolicyError::EraChanged { current: version })) + if version == current.history.version + )); + + // Every backup is evaluated under its own saved generation. Latest-cut-only + // matching would incorrectly resurrect the discarded g0 transfer at offset 3. + let mut selected = None; + for (candidate, eligible, cut) in [ + (&good_zero, true, 3), + (&bad_zero, false, 3), + (&middle_one, true, 5), + (&good_one, true, 5), + (&bad_one, false, 5), + ] { + let info = client + .history(Some(era), Some(candidate.claim.version.recovery_generation)) + .await + .unwrap(); + let compatibility = info.compatibility.unwrap(); + assert_eq!( + compatibility.from_generation, + candidate.claim.version.recovery_generation + ); + assert_eq!(compatibility.preserved_input_count.get(), cut); + let survives = candidate.claim.next_input >= info.history.available_from + && candidate.claim.next_input <= compatibility.preserved_input_count; + assert_eq!(survives, eligible); + if survives + && selected.is_none_or(|previous: &Backup| { + previous.claim.next_input < candidate.claim.next_input + }) + { + selected = Some(candidate); + } + } + let selected = selected.unwrap(); + assert_eq!(selected.claim, good_one.claim); + let (mut reader_app, mut reader_history) = selected.restore(); + let mut stream = client + .subscribe(HistoryClaim { + version: current.history.version, + next_input: selected.claim.next_input, + }) + .await + .unwrap(); + for _ in 5..8 { + replay_one(&mut stream, &mut reader_app, &mut reader_history).await; + } + assert_eq!( + reader_app.canonical_snapshot_bytes().unwrap(), + app.canonical_snapshot_bytes().unwrap() + ); + assert_eq!( + reader_history, history, + "restored projection loses invalidated transfers and follows their replacements" + ); + drop(stream); + + let lookup = client + .history(Some(era), Some(good_one.claim.version.recovery_generation)) + .await + .unwrap(); + assert_eq!(lookup.compatibility.unwrap().preserved_input_count.get(), 5); + recover_tip(&mut storage, &mut head, 4500); + let stale = client + .subscribe(HistoryClaim { + version: lookup.history.version, + next_input: good_one.claim.next_input, + }) + .await; + assert!( + matches!(stale, Err(SubscribeError::History(HistoryPolicyError::StaleGeneration { current })) if current.recovery_generation.get() == 3) + ); + let fresh = client + .history(Some(era), Some(good_one.claim.version.recovery_generation)) + .await + .unwrap(); + assert_eq!(fresh.compatibility.unwrap().preserved_input_count.get(), 5); + (reader_app, reader_history) = good_one.restore(); + (app, history) = good_one.restore(); + let mut stream = client + .subscribe(HistoryClaim { + version: fresh.history.version, + next_input: good_one.claim.next_input, + }) + .await + .unwrap(); + append_transfer(&mut storage, &mut head, &mut app, &mut history, &signer, 40); + replay_one(&mut stream, &mut reader_app, &mut reader_history).await; + assert_eq!( + reader_app.canonical_snapshot_bytes().unwrap(), + app.canonical_snapshot_bytes().unwrap() + ); + assert_eq!(reader_history, history); + drop(stream); + server.stop().await; +} diff --git a/sequencer/src/integration_tests/mod.rs b/sequencer/src/integration_tests/mod.rs index 8319d5f..3aba7ce 100644 --- a/sequencer/src/integration_tests/mod.rs +++ b/sequencer/src/integration_tests/mod.rs @@ -5,5 +5,6 @@ mod batch_submitter; mod chain_id_validation; mod common; mod e2e_sequencer; +mod historical_bootstrap; mod snapshot_endpoints; mod ws_broadcaster; diff --git a/sequencer/src/storage/egress.rs b/sequencer/src/storage/egress.rs index bb0f44b..2f64adb 100644 --- a/sequencer/src/storage/egress.rs +++ b/sequencer/src/storage/egress.rs @@ -1,7 +1,7 @@ // (c) Cartesi and individual authors (see AUTHORS) // SPDX-License-Identifier: Apache-2.0 (see LICENSE) -//! Application replay entries shared by catch-up and consumer history reads. +//! Application replay entries and the historical L1 prefix for consumer bootstrap. use alloy_primitives::{Address, B256}; use rusqlite::{Result, Row}; @@ -12,6 +12,8 @@ use super::convert::{i64_to_u16, i64_to_u32, i64_to_u64}; mod canonical; pub(crate) use canonical::HistoryReadError; +mod historical; +pub(crate) use historical::HistoricalReadError; #[derive(Debug, Clone)] pub(crate) enum L2TxContext { diff --git a/sequencer/src/storage/egress/historical.rs b/sequencer/src/storage/egress/historical.rs new file mode 100644 index 0000000..730879a --- /dev/null +++ b/sequencer/src/storage/egress/historical.rs @@ -0,0 +1,262 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +//! Era-pinned raw L1 pages and their immutable handoff to application history. + +use alloy_primitives::{Address, B256}; +use rusqlite::{Connection, OptionalExtension, params}; +use sequencer_core::history::{ + EraId, ExecutedInputCount, HistoryBounds, HistoryPolicyError, RecoveryGeneration, +}; +use sequencer_core::history_api::{ + AcceptedCheckpoint, HISTORICAL_INPUT_MAX_ITEMS, HISTORICAL_INPUT_PAYLOAD_TARGET_BYTES, + HistoricalL1Input, HistoricalL1InputStart, HistoricalL1InputsPage, HistoryBaseline, + HistoryCompatibility, HistoryDeployment, HistoryInfo, +}; + +use crate::storage::Storage; +use crate::storage::convert::{i64_to_u64, u64_to_i64}; +use crate::storage::history::{ + next_executed_input_count_in, preserved_input_count_in, query_history_state, +}; +use crate::storage::l1_inputs::query_deployment_identity; +use crate::storage::mutations::batch_tree_anchor_in; +use crate::storage::safe_accepted_batches::canonical_divergence_in; +use crate::storage::snapshot_dumps::{finalized_dump_in, has_rollback_safe_snapshot_in}; + +#[derive(Debug, thiserror::Error)] +pub(crate) enum HistoricalReadError { + #[error(transparent)] + Policy(#[from] HistoryPolicyError), + #[error("{0}")] + BadRequest(String), + #[error("canonical divergence prevents accepted checkpoint selection")] + CanonicalDivergence, + #[error("reading historical L1 inputs: {0}")] + Storage(#[from] rusqlite::Error), +} + +impl Storage { + pub(crate) fn history_info( + &mut self, + expected_era: Option, + from_generation: Option, + ) -> Result { + self.read(|tx| { + let state = query_history_state(tx)?; + if expected_era.is_some_and(|era| era != state.version.era_id) { + return Ok(Err(HistoryPolicyError::EraChanged { + current: state.version, + } + .into())); + } + if from_generation.is_some() && expected_era.is_none() { + return Ok(Err(HistoricalReadError::BadRequest( + "era_id is required with from_generation".to_owned(), + ))); + } + if from_generation.is_some_and(|from| from > state.version.recovery_generation) { + return Ok(Err(HistoricalReadError::BadRequest( + "from_generation exceeds the current recovery generation".to_owned(), + ))); + } + if canonical_divergence_in(tx)?.is_some() { + return Ok(Err(HistoricalReadError::CanonicalDivergence)); + } + let deployment = + query_deployment_identity(tx)?.ok_or(rusqlite::Error::QueryReturnedNoRows)?; + let accepted = finalized_dump_in(tx)?; + if accepted.is_none() { + assert!( + has_rollback_safe_snapshot_in(tx)?, + "history has no rollback-safe snapshot" + ); + } + let head = next_executed_input_count_in(tx)?; + let compatibility = from_generation + .map(|from| { + preserved_input_count_in(tx, from, state.version.recovery_generation, head).map( + |preserved_input_count| HistoryCompatibility { + from_generation: from, + preserved_input_count, + }, + ) + }) + .transpose()?; + Ok(Ok(HistoryInfo { + deployment: HistoryDeployment { + chain_id: deployment.chain_id, + app_address: deployment.app_address, + input_box_address: deployment.input_box_address, + app_deployment_block: deployment.app_deployment_block, + batch_submitter_address: deployment.batch_submitter_address, + }, + history: HistoryBounds { + version: state.version, + available_from: ExecutedInputCount::new(state.base_executed_input_count), + head, + }, + baseline: HistoryBaseline { + l1_stop_block: state.base_safe_block, + l1_end_input_index: historical_end_in(tx, state.base_safe_block)?, + next_batch_nonce: batch_tree_anchor_in(tx)?, + }, + accepted_checkpoint: accepted.map(|snapshot| AcceptedCheckpoint { + inclusion_block: snapshot.inclusion_block, + executed_input_count: snapshot.executed_input_count, + next_batch_nonce: snapshot.next_batch_nonce, + }), + compatibility, + })) + })? + } + + pub(crate) fn historical_l1_inputs( + &mut self, + era: EraId, + start: HistoricalL1InputStart, + limit: usize, + ) -> Result { + self.read(|tx| { + let state = query_history_state(tx)?; + if era != state.version.era_id { + return Ok(Err(HistoryPolicyError::EraChanged { + current: state.version, + } + .into())); + } + let end = historical_end_in(tx, state.base_safe_block)?; + if !(1..=HISTORICAL_INPUT_MAX_ITEMS).contains(&limit) { + return Ok(Err(HistoricalReadError::BadRequest(format!( + "limit must be between 1 and {HISTORICAL_INPUT_MAX_ITEMS}" + )))); + } + let next = match start { + HistoricalL1InputStart::NextInputIndex(next) if next <= end => next, + HistoricalL1InputStart::AfterBlock(block) if block <= state.base_safe_block => { + first_input_after_block_in(tx, block, state.base_safe_block)?.unwrap_or(end) + } + HistoricalL1InputStart::NextInputIndex(_) => { + return Ok(Err(HistoricalReadError::BadRequest(format!( + "next_input_index exceeds historical end {end}" + )))); + } + HistoricalL1InputStart::AfterBlock(_) => { + return Ok(Err(HistoricalReadError::BadRequest(format!( + "after_block exceeds historical stop block {}", + state.base_safe_block + )))); + } + }; + raw_page_in(tx, era, state.base_safe_block, end, next, limit).map(Ok) + })? + } +} + +fn historical_end_in(conn: &Connection, stop: u64) -> rusqlite::Result { + let last: Option = conn + .query_row( + "SELECT safe_input_index FROM safe_inputs WHERE block_number <= ?1 \ + ORDER BY block_number DESC, safe_input_index DESC LIMIT 1", + [u64_to_i64(stop)], + |row| row.get(0), + ) + .optional()?; + Ok(last.map_or(0, |index| { + i64_to_u64(index) + .checked_add(1) + .expect("historical input index overflow") + })) +} + +fn first_input_after_block_in( + conn: &Connection, + block: u64, + stop: u64, +) -> rusqlite::Result> { + conn.query_row( + "SELECT safe_input_index FROM safe_inputs WHERE block_number > ?1 AND block_number <= ?2 \ + ORDER BY block_number, safe_input_index LIMIT 1", + params![u64_to_i64(block), u64_to_i64(stop)], + |row| Ok(i64_to_u64(row.get(0)?)), + ) + .optional() +} + +fn raw_page_in( + conn: &Connection, + era_id: EraId, + stop: u64, + end: u64, + mut next: u64, + limit: usize, +) -> rusqlite::Result { + let expected_len = (end - next).min(limit as u64); + let mut items = Vec::new(); + if expected_len > 0 { + let mut stmt = conn.prepare_cached( + "SELECT safe_input_index, sender, payload, block_number, block_timestamp, \ + transaction_hash, length(payload) \ + FROM safe_inputs WHERE safe_input_index >= ?1 AND safe_input_index <= ?2 \ + ORDER BY safe_input_index LIMIT ?3", + )?; + let mut rows = stmt.query(params![ + u64_to_i64(next), + u64_to_i64(end - 1), + u64_to_i64(expected_len), + ])?; + let mut payload_bytes = 0_u64; + let mut byte_limited = false; + while let Some(row) = rows.next()? { + let input_index = i64_to_u64(row.get(0)?); + assert_eq!(input_index, next, "historical L1 page has an input gap"); + let payload_len = i64_to_u64(row.get(6)?); + // Budget before copying the BLOB out of SQLite. The first row may + // exceed the target so every valid L1 input remains consumable. + if !items.is_empty() + && payload_len > (HISTORICAL_INPUT_PAYLOAD_TARGET_BYTES as u64 - payload_bytes) + { + byte_limited = true; + break; + } + let block_number = i64_to_u64(row.get(3)?); + assert!( + block_number <= stop, + "historical L1 page exceeds its stop block" + ); + items.push(HistoricalL1Input { + input_index, + sender: Address::from_slice(&row.get::<_, Vec>(1)?), + payload: row.get::<_, Vec>(2)?.into(), + block_number, + block_timestamp: i64_to_u64(row.get(4)?), + transaction_hash: B256::from_slice(&row.get::<_, Vec>(5)?), + }); + next = next + .checked_add(1) + .expect("historical input index overflow"); + payload_bytes += payload_len; + if payload_bytes > HISTORICAL_INPUT_PAYLOAD_TARGET_BYTES as u64 { + byte_limited = true; + break; + } + } + if !byte_limited { + assert_eq!( + items.len() as u64, + expected_len, + "historical L1 page ended before its recorded end" + ); + } + } + Ok(HistoricalL1InputsPage { + era_id, + l1_stop_block: stop, + end_input_index: end, + next_input_index: next, + items, + }) +} + +#[cfg(test)] +mod tests; diff --git a/sequencer/src/storage/egress/historical/tests.rs b/sequencer/src/storage/egress/historical/tests.rs new file mode 100644 index 0000000..cf1587d --- /dev/null +++ b/sequencer/src/storage/egress/historical/tests.rs @@ -0,0 +1,495 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +use std::path::Path; + +use super::*; +use crate::storage::history::{advance_recovery_generation_in, initialize_history_in}; +use crate::storage::test_helpers::{ + SENDER_A, SENDER_B, TestDb, default_protocol_timing, local_batch_payload, + pin_test_deployment_identity, temp_db, +}; +use crate::storage::{ + DirectInputExecution, FrontierMode, IngestedSafeInput, LifecycleCommand, SafeInputRange, +}; + +fn fixture(name: &str, stop: u64, count: u64, nonce: u64) -> (TestDb, Storage) { + let db = temp_db(name); + let mut storage = Storage::initialize_for_command(&db.path, LifecycleCommand::Rebuild).unwrap(); + pin_test_deployment_identity(&mut storage, SENDER_A); + storage + .write(|tx| initialize_history_in(tx, ExecutedInputCount::new(count), stop)) + .unwrap(); + storage.set_batch_tree_anchor(nonce).unwrap(); + storage + .insert_baseline_snapshot( + Path::new("/snapshot/baseline"), + ExecutedInputCount::new(count), + ) + .unwrap(); + (db, storage) +} + +fn input(block: u64, sender: Address, payload: Vec) -> IngestedSafeInput { + IngestedSafeInput { + sender, + payload, + block_number: block, + block_timestamp: block * 12, + transaction_hash: B256::repeat_byte(block as u8), + } +} + +fn append(storage: &mut Storage, safe_block: u64, inputs: &[IngestedSafeInput]) { + storage + .append_ingested_safe_inputs_with_timestamp( + safe_block, + safe_block * 12, + inputs, + SENDER_A, + &default_protocol_timing(), + FrontierMode::DeferUntilAnchorSet, + ) + .unwrap(); +} + +fn era(storage: &Storage) -> EraId { + storage.history_state().unwrap().version.era_id +} + +fn page(storage: &mut Storage, next: u64, limit: usize) -> HistoricalL1InputsPage { + storage + .historical_l1_inputs( + era(storage), + HistoricalL1InputStart::NextInputIndex(next), + limit, + ) + .unwrap() +} + +#[test] +fn genesis_history_has_an_empty_raw_prefix_even_after_live_inputs_arrive() { + let (_db, mut storage) = fixture("historical-genesis", 0, 0, 0); + append(&mut storage, 20, &[input(10, SENDER_B, vec![1])]); + let info = storage.history_info(None, None).unwrap(); + assert_eq!(info.history.available_from, ExecutedInputCount::ZERO); + assert_eq!(info.history.head, ExecutedInputCount::ZERO); + assert_eq!(info.compatibility, None); + assert_eq!( + info.baseline, + HistoryBaseline { + l1_stop_block: 0, + l1_end_input_index: 0, + next_batch_nonce: 0 + } + ); + assert_eq!( + info.accepted_checkpoint, + Some(AcceptedCheckpoint { + inclusion_block: 0, + executed_input_count: ExecutedInputCount::ZERO, + next_batch_nonce: 0, + }) + ); + assert_eq!(info.deployment.chain_id, 1); + assert_eq!(info.deployment.app_address, Address::repeat_byte(0x11)); + assert_eq!( + info.deployment.input_box_address, + Address::repeat_byte(0x22) + ); + assert_eq!(info.deployment.app_deployment_block, 0); + assert_eq!(info.deployment.batch_submitter_address, SENDER_A); + let raw = page(&mut storage, 0, 10); + assert!(raw.items.is_empty()); + assert_eq!((raw.end_input_index, raw.next_input_index), (0, 0)); +} + +#[test] +fn rebuilt_history_preserves_raw_payloads_metadata_and_its_fixed_prefix() { + let (_db, mut storage) = fixture("historical-rebuilt", 20, 41, 7); + let inputs = [ + input(5, SENDER_B, vec![0xaa, 0xbb]), + input(12, SENDER_A, vec![0xff]), // Deliberately malformed batch bytes. + input(20, SENDER_B, vec![]), + input(21, SENDER_B, vec![0xcc]), + ]; + append(&mut storage, 25, &inputs); + let info = storage.history_info(None, None).unwrap(); + assert_eq!(info.history.available_from, ExecutedInputCount::new(41)); + assert_eq!(info.history.head, ExecutedInputCount::new(41)); + assert_eq!(info.baseline.l1_stop_block, 20); + assert_eq!(info.baseline.l1_end_input_index, 3); + assert_eq!(info.baseline.next_batch_nonce, 7); + assert_eq!(info.accepted_checkpoint, None); + + let first = page(&mut storage, 0, 2); + assert_eq!((first.end_input_index, first.next_input_index), (3, 2)); + for (index, actual) in first.items.iter().enumerate() { + let expected = &inputs[index]; + assert_eq!(actual.input_index, index as u64); + assert_eq!(actual.sender, expected.sender); + assert_eq!(actual.payload.as_ref(), expected.payload); + assert_eq!(actual.block_number, expected.block_number); + assert_eq!(actual.block_timestamp, expected.block_timestamp); + assert_eq!(actual.transaction_hash, expected.transaction_hash); + } + let last = page(&mut storage, first.next_input_index, 2); + assert_eq!(last.items.len(), 1); + assert_eq!(last.items[0].input_index, 2); + assert!(last.items[0].payload.is_empty()); + assert_eq!((last.end_input_index, last.next_input_index), (3, 3)); + assert!(page(&mut storage, 3, 2).items.is_empty()); + + append(&mut storage, 40, &[input(35, SENDER_B, vec![9])]); + storage.write(advance_recovery_generation_in).unwrap(); + assert_eq!(page(&mut storage, 0, 2), first); + let updated = storage + .history_info(Some(info.history.version.era_id), None) + .unwrap(); + assert_eq!(updated.baseline, info.baseline); + assert_eq!(updated.history.version.recovery_generation.get(), 1); +} + +#[test] +fn after_block_skips_the_whole_block_and_paging_keeps_its_remaining_rows() { + let (_db, mut storage) = fixture("historical-block-seek", 30, 9, 2); + append( + &mut storage, + 40, + &[ + input(5, SENDER_B, vec![0]), + input(12, SENDER_A, vec![1]), + input(12, SENDER_B, vec![2]), + input(20, SENDER_B, vec![3]), + input(20, SENDER_A, vec![4]), + input(31, SENDER_B, vec![5]), + ], + ); + let current = era(&storage); + let first = storage + .historical_l1_inputs(current, HistoricalL1InputStart::AfterBlock(12), 1) + .unwrap(); + assert_eq!(first.items[0].input_index, 3); + assert_eq!(first.next_input_index, 4); + assert_eq!(page(&mut storage, 4, 1).items[0].input_index, 4); + for block in [20, 25, 30] { + let eof = storage + .historical_l1_inputs(current, HistoricalL1InputStart::AfterBlock(block), 1) + .unwrap(); + assert!(eof.items.is_empty()); + assert_eq!((eof.end_input_index, eof.next_input_index), (5, 5)); + } +} + +#[test] +fn era_validation_precedes_numeric_bounds_even_for_empty_history() { + let (_db, mut storage) = fixture("historical-claim", 0, 0, 0); + let current = storage.history_state().unwrap().version; + let other: EraId = "00112233-4455-4677-8899-aabbccddeeff".parse().unwrap(); + assert_ne!(other, current.era_id); + for start in [ + HistoricalL1InputStart::NextInputIndex(0), + HistoricalL1InputStart::NextInputIndex(u64::MAX), + HistoricalL1InputStart::AfterBlock(u64::MAX), + ] { + for limit in [0, 1, usize::MAX] { + assert!(matches!( + storage.historical_l1_inputs(other, start, limit), + Err(HistoricalReadError::Policy(HistoryPolicyError::EraChanged { current: actual })) + if actual == current + )); + } + } + assert!(matches!( + storage.history_info(Some(other), None), + Err(HistoricalReadError::Policy(HistoryPolicyError::EraChanged { current: actual })) + if actual == current + )); + for (start, limit) in [ + (HistoricalL1InputStart::NextInputIndex(0), 0), + ( + HistoricalL1InputStart::NextInputIndex(0), + HISTORICAL_INPUT_MAX_ITEMS + 1, + ), + (HistoricalL1InputStart::NextInputIndex(1), 1), + (HistoricalL1InputStart::NextInputIndex(u64::MAX), 1), + (HistoricalL1InputStart::AfterBlock(1), 1), + (HistoricalL1InputStart::AfterBlock(u64::MAX), 1), + ] { + assert!(matches!( + storage.historical_l1_inputs(current.era_id, start, limit), + Err(HistoricalReadError::BadRequest(_)) + )); + } +} + +#[test] +fn byte_budget_makes_progress_through_oversized_inputs_without_skipping_them() { + let (_db, mut storage) = fixture("historical-byte-budget", 20, 1, 1); + let target = HISTORICAL_INPUT_PAYLOAD_TARGET_BYTES; + append( + &mut storage, + 20, + &[ + input(1, SENDER_B, vec![1; target / 2]), + input(2, SENDER_B, vec![2; target / 2 + 1]), + input(3, SENDER_A, vec![3; target + 1]), + input(4, SENDER_B, vec![4]), + ], + ); + for (start, expected_len) in [ + (0, target / 2), + (1, target / 2 + 1), + (2, target + 1), + (3, 1), + ] { + let raw = page(&mut storage, start, HISTORICAL_INPUT_MAX_ITEMS); + assert_eq!(raw.items.len(), 1); + assert_eq!(raw.items[0].input_index, start); + assert_eq!(raw.items[0].payload.len(), expected_len); + assert_eq!(raw.next_input_index, start + 1); + assert_eq!(raw.end_input_index, 4); + } +} + +#[test] +fn exact_byte_target_and_item_cap_have_independent_boundaries() { + let (_db, mut storage) = fixture("historical-item-budget", 20, 1, 1); + let mut inputs = vec![input( + 1, + SENDER_B, + vec![7; HISTORICAL_INPUT_PAYLOAD_TARGET_BYTES], + )]; + inputs.extend((0..HISTORICAL_INPUT_MAX_ITEMS).map(|_| input(2, SENDER_B, vec![]))); + append(&mut storage, 20, &inputs); + let first = page(&mut storage, 0, HISTORICAL_INPUT_MAX_ITEMS); + assert_eq!(first.items.len(), HISTORICAL_INPUT_MAX_ITEMS); + assert_eq!(first.next_input_index, HISTORICAL_INPUT_MAX_ITEMS as u64); + assert_eq!(first.end_input_index, HISTORICAL_INPUT_MAX_ITEMS as u64 + 1); + let last = page( + &mut storage, + first.next_input_index, + HISTORICAL_INPUT_MAX_ITEMS, + ); + assert_eq!(last.items.len(), 1); + assert_eq!(last.next_input_index, last.end_input_index); +} + +#[test] +#[should_panic(expected = "historical L1 page has an input gap")] +fn missing_raw_input_is_an_invariant_fault() { + let (_db, mut storage) = fixture("historical-gap", 20, 3, 1); + append( + &mut storage, + 20, + &[ + input(1, SENDER_B, vec![1]), + input(2, SENDER_B, vec![2]), + input(3, SENDER_B, vec![3]), + ], + ); + storage + .conn + .execute("DELETE FROM safe_inputs WHERE safe_input_index=1", []) + .unwrap(); + let _ = page(&mut storage, 0, 3); +} + +#[test] +fn maximum_sqlite_index_has_an_unclamped_exclusive_end() { + let (_db, mut storage) = fixture("historical-max-index", 20, 1, 1); + storage.conn.execute( + "INSERT INTO safe_inputs (safe_input_index,sender,payload,block_number,block_timestamp,transaction_hash) \ + VALUES (?1,?2,?3,20,240,?4)", + params![i64::MAX, SENDER_B.as_slice(), &[1_u8][..], B256::ZERO.as_slice()], + ).unwrap(); + let max = i64::MAX as u64; + let last = page(&mut storage, max, 10); + assert_eq!(last.items.len(), 1); + assert_eq!( + (last.end_input_index, last.next_input_index), + (max + 1, max + 1) + ); + assert!(page(&mut storage, max + 1, 10).items.is_empty()); +} + +#[test] +fn accepted_checkpoint_uses_the_exact_latest_snapshot_and_preserves_baseline() { + let (_db, mut storage) = fixture("historical-accepted", 20, 41, 7); + let mut head = storage + .initialize_open_state(20, SafeInputRange::empty_at(0)) + .unwrap(); + for index in 0..2 { + storage + .close_frame_and_batch_with_snapshot( + &mut head, + 20, + Path::new(&format!("/snapshot/{index}")), + index, + ExecutedInputCount::new(41), + ) + .unwrap(); + } + let payloads = [ + local_batch_payload(&mut storage, 7), + local_batch_payload(&mut storage, 8), + ]; + storage + .append_ingested_safe_inputs_with_timestamp( + 30, + 360, + &[ + input(29, SENDER_A, payloads[0].clone()), + input(30, SENDER_A, payloads[1].clone()), + ], + SENDER_A, + &default_protocol_timing(), + FrontierMode::Populate, + ) + .unwrap(); + storage.gc_unreferenced_dumps().unwrap(); + let info = storage.history_info(None, None).unwrap(); + assert_eq!(info.baseline.next_batch_nonce, 7); + assert_eq!(info.baseline.l1_stop_block, 20); + assert_eq!(info.baseline.l1_end_input_index, 0); + assert_eq!( + info.accepted_checkpoint, + Some(AcceptedCheckpoint { + inclusion_block: 30, + executed_input_count: ExecutedInputCount::new(41), + next_batch_nonce: 9, + }) + ); + storage + .conn + .execute("DELETE FROM snapshots WHERE batch_index=1", []) + .unwrap(); + assert!(matches!( + storage.history_info(None, None), + Err(HistoricalReadError::Storage( + rusqlite::Error::QueryReturnedNoRows + )) + )); +} + +#[test] +fn canonical_divergence_cannot_be_advertised_as_an_accepted_receipt() { + let (_db, mut storage) = fixture("historical-divergence", 0, 0, 0); + storage + .conn + .execute( + "INSERT INTO canonical_divergence \ + (singleton_id,nonce,safe_input_index,kind,detected_at_ms) VALUES (0,0,0,'foreign',0)", + [], + ) + .unwrap(); + assert!(matches!( + storage.history_info(None, None), + Err(HistoricalReadError::CanonicalDivergence) + )); +} + +#[test] +#[should_panic(expected = "history has no rollback-safe snapshot")] +fn missing_baseline_is_not_reported_as_an_absent_accepted_checkpoint() { + let (_db, mut storage) = fixture("historical-missing-baseline", 20, 41, 7); + storage.conn.execute("DELETE FROM snapshots", []).unwrap(); + let _ = storage.history_info(None, None); +} + +#[test] +fn compatibility_requires_an_era_and_rejects_future_generations_after_era_validation() { + let (_db, mut storage) = fixture("historical-generation-query", 0, 0, 0); + let current = era(&storage); + assert!(matches!( + storage.history_info(None, Some(RecoveryGeneration::new(0))), + Err(HistoricalReadError::BadRequest(_)) + )); + for from in [1, u64::MAX] { + assert!(matches!( + storage.history_info(Some(current), Some(RecoveryGeneration::new(from))), + Err(HistoricalReadError::BadRequest(_)) + )); + } + let other = "00112233-4455-4677-8899-aabbccddeeff".parse().unwrap(); + assert!(matches!( + storage.history_info(Some(other), Some(RecoveryGeneration::new(u64::MAX))), + Err(HistoricalReadError::Policy( + HistoryPolicyError::EraChanged { .. } + )) + )); + let info = storage + .history_info(Some(current), Some(RecoveryGeneration::new(0))) + .unwrap(); + assert_eq!( + info.compatibility, + Some(HistoryCompatibility { + from_generation: RecoveryGeneration::new(0), + preserved_input_count: ExecutedInputCount::ZERO, + }) + ); +} + +#[test] +fn full_recovery_preserves_nonzero_baseline_before_replacement_directs() { + let (_db, mut storage) = fixture("historical-generation-baseline", 100, 41, 7); + let mut head = storage + .initialize_open_state(100, SafeInputRange::empty_at(0)) + .unwrap(); + let now = crate::clock::unix_now_ms(); + let protocol = default_protocol_timing(); + storage + .append_ingested_safe_inputs_with_timestamp( + 1400, + now / 1000, + &[input(110, SENDER_B, vec![1])], + SENDER_A, + &protocol, + FrontierMode::Populate, + ) + .unwrap(); + storage + .close_frame_only_with_executions( + &mut head, + 110, + SafeInputRange::new(0, 1), + &[DirectInputExecution { + safe_input_index: 0, + executed_input_offset: ExecutedInputCount::new(41), + }], + ) + .unwrap(); + let current = era(&storage); + let before = storage + .history_info(Some(current), Some(RecoveryGeneration::new(0))) + .unwrap(); + assert_eq!(before.history.head, ExecutedInputCount::new(42)); + assert_eq!( + before.compatibility.unwrap().preserved_input_count, + before.history.head + ); + assert_eq!( + storage + .recover_aging_tip_for_recovery(head.batch_index, &protocol, now) + .unwrap(), + [0] + ); + + let after = storage + .history_info(Some(current), Some(RecoveryGeneration::new(0))) + .unwrap(); + assert_eq!(after.history.available_from, ExecutedInputCount::new(41)); + assert_eq!(after.history.head, ExecutedInputCount::new(42)); + assert_eq!(after.history.version.recovery_generation.get(), 1); + assert_eq!( + after.compatibility.unwrap().preserved_input_count, + ExecutedInputCount::new(41) + ); + let latest = storage + .history_info(Some(current), Some(RecoveryGeneration::new(1))) + .unwrap(); + assert_eq!( + latest.compatibility.unwrap().preserved_input_count, + latest.history.head + ); +} diff --git a/sequencer/src/storage/history.rs b/sequencer/src/storage/history.rs index 65f24f3..0948d1d 100644 --- a/sequencer/src/storage/history.rs +++ b/sequencer/src/storage/history.rs @@ -1,7 +1,7 @@ // (c) Cartesi and individual authors (see AUTHORS) // SPDX-License-Identifier: Apache-2.0 (see LICENSE) -//! Immutable era baseline and current application-history generation. +//! Immutable era baseline and the preserved prefix at each recovery generation. #[cfg(test)] use rusqlite::OptionalExtension; @@ -112,11 +112,18 @@ pub(super) fn next_executed_input_count_in(conn: &Connection) -> Result) -> Result { let current = query_history_state(tx)?.version.recovery_generation.get(); let next = current .checked_add(1) .expect("recovery generation exhausted"); + let preserved = next_executed_input_count_in(tx)?; + tx.execute( + "INSERT INTO history_generation_cuts (recovery_generation, preserved_input_count) \ + VALUES (?1, ?2)", + params![u64_to_i64(next), u64_to_i64(preserved.get())], + )?; let changed = tx.execute( "UPDATE history_state SET recovery_generation = ?1 WHERE singleton_id = 0", [u64_to_i64(next)], @@ -127,6 +134,36 @@ pub(super) fn advance_recovery_generation_in(tx: &Transaction<'_>) -> Result Result { + assert!( + from <= current, + "compatibility starts after the current generation" + ); + if from == current { + return Ok(head); + } + let (count, minimum): (i64, Option) = conn.query_row( + "SELECT COUNT(*), MIN(preserved_input_count) FROM history_generation_cuts \ + WHERE recovery_generation > ?1 AND recovery_generation <= ?2", + params![u64_to_i64(from.get()), u64_to_i64(current.get())], + |row| Ok((row.get(0)?, row.get(1)?)), + )?; + // Unique integer generations plus the exact interval length prove that + // every intervening recovery contributed its cut, including empty batches. + assert_eq!( + i64_to_u64(count), + current.get() - from.get(), + "history generation lineage is incomplete" + ); + let minimum = minimum.expect("a nonempty complete generation interval has a minimum"); + Ok(head.min(ExecutedInputCount::new(i64_to_u64(minimum)))) +} + #[cfg(test)] mod tests { use super::*; @@ -194,3 +231,6 @@ mod tests { ); } } + +#[cfg(test)] +mod generation_tests; diff --git a/sequencer/src/storage/history/generation_tests.rs b/sequencer/src/storage/history/generation_tests.rs new file mode 100644 index 0000000..977db21 --- /dev/null +++ b/sequencer/src/storage/history/generation_tests.rs @@ -0,0 +1,170 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +use super::*; +use crate::storage::test_helpers::temp_db; + +fn ledger(storage: &Storage) -> Vec<(i64, i64)> { + storage + .conn + .prepare("SELECT recovery_generation, preserved_input_count FROM history_generation_cuts ORDER BY recovery_generation") + .unwrap() + .query_map([], |row| Ok((row.get(0)?, row.get(1)?))) + .unwrap() + .collect::>() + .unwrap() +} + +// Synthetic cuts exercise the query's full interval contract independently +// of today's recovery pivot policy. Actual cascades are tested separately. +fn seed_cuts(storage: &mut Storage, cuts: &[u64]) { + storage + .write(|tx| { + for (index, cut) in cuts.iter().enumerate() { + let generation = i64::try_from(index + 1).unwrap(); + tx.execute( + "INSERT INTO history_generation_cuts (recovery_generation,preserved_input_count) VALUES (?1,?2)", + params![generation, u64_to_i64(*cut)], + )?; + tx.execute( + "UPDATE history_state SET recovery_generation=?1 WHERE singleton_id=0", + [generation], + )?; + } + Ok(()) + }) + .unwrap(); +} + +#[test] +fn generation_and_cut_are_atomic_and_immutable() { + let db = temp_db("generation-cut-atomic"); + let mut storage = Storage::open(&db.path).unwrap(); + let result: Result<()> = storage.write(|tx| { + advance_recovery_generation_in(tx)?; + Err(rusqlite::Error::InvalidQuery) + }); + assert!(result.is_err()); + assert!(ledger(&storage).is_empty()); + assert_eq!( + storage + .history_state() + .unwrap() + .version + .recovery_generation + .get(), + 0 + ); + assert!( + storage + .conn + .execute("UPDATE history_state SET recovery_generation=1", []) + .is_err() + ); + + storage.write(advance_recovery_generation_in).unwrap(); + assert_eq!(ledger(&storage), [(1, 0)]); + for sql in [ + "UPDATE history_generation_cuts SET preserved_input_count=1", + "UPDATE history_generation_cuts SET recovery_generation=2", + "DELETE FROM history_generation_cuts", + "INSERT INTO history_generation_cuts VALUES (1,1)", + "INSERT INTO history_generation_cuts VALUES (0,0)", + "INSERT INTO history_generation_cuts VALUES (2,-1)", + ] { + assert!(storage.conn.execute(sql, []).is_err(), "{sql}"); + } + drop(storage); + let reopened = Storage::open(&db.path).unwrap(); + assert_eq!(ledger(&reopened), [(1, 0)]); + assert_eq!( + reopened + .history_state() + .unwrap() + .version + .recovery_generation + .get(), + 1 + ); +} + +#[test] +fn compatibility_uses_every_intervening_cut_and_the_current_head() { + let db = temp_db("generation-cut-minimum"); + let mut storage = Storage::open(&db.path).unwrap(); + seed_cuts(&mut storage, &[3, 5, 2, 4]); + for (from, current, head, expected) in [ + (0, 1, 9, 3), + (0, 2, 9, 3), + (1, 2, 9, 5), + (0, 3, 9, 2), + (1, 3, 9, 2), + (2, 3, 9, 2), + (0, 4, 9, 2), + (3, 4, 9, 4), + (3, 4, 1, 1), + (4, 4, 9, 9), + ] { + assert_eq!( + preserved_input_count_in( + &storage.conn, + RecoveryGeneration::new(from), + RecoveryGeneration::new(current), + ExecutedInputCount::new(head) + ) + .unwrap(), + ExecutedInputCount::new(expected), + "from={from}, current={current}, head={head}", + ); + } +} + +#[test] +#[should_panic(expected = "history generation lineage is incomplete")] +fn missing_intermediate_cut_fails_instead_of_certifying_a_partial_minimum() { + let db = temp_db("generation-cut-gap"); + let mut storage = Storage::open(&db.path).unwrap(); + seed_cuts(&mut storage, &[3, 1, 5]); + storage + .conn + .execute_batch( + "DROP TRIGGER trg_history_generation_cuts_not_deletable; \ + DELETE FROM history_generation_cuts WHERE recovery_generation=2", + ) + .unwrap(); + let _ = preserved_input_count_in( + &storage.conn, + RecoveryGeneration::new(0), + RecoveryGeneration::new(3), + ExecutedInputCount::new(9), + ); +} + +#[test] +fn current_head_may_be_the_boundary_after_the_maximum_sqlite_offset() { + let db = temp_db("generation-cut-max-head"); + let mut storage = Storage::open(&db.path).unwrap(); + let max = i64::MAX as u64; + seed_cuts(&mut storage, &[max]); + let head = ExecutedInputCount::new(max + 1); + assert_eq!( + preserved_input_count_in( + &storage.conn, + RecoveryGeneration::new(1), + RecoveryGeneration::new(1), + head + ) + .unwrap(), + head, + ); + assert_eq!( + preserved_input_count_in( + &storage.conn, + RecoveryGeneration::new(0), + RecoveryGeneration::new(1), + head + ) + .unwrap(), + ExecutedInputCount::new(max), + ); +} diff --git a/sequencer/src/storage/migrations/0001_schema.sql b/sequencer/src/storage/migrations/0001_schema.sql index a9650de..f93e691 100644 --- a/sequencer/src/storage/migrations/0001_schema.sql +++ b/sequencer/src/storage/migrations/0001_schema.sql @@ -376,6 +376,19 @@ CREATE TABLE IF NOT EXISTS history_state ( base_safe_block INTEGER NOT NULL CHECK ( typeof(base_safe_block) = 'integer' AND base_safe_block >= 0) ); +-- One cut for each transition, measured after invalidation removes the old +-- suffix and before reopening attributes replacement direct inputs. +CREATE TABLE IF NOT EXISTS history_generation_cuts ( + recovery_generation INTEGER PRIMARY KEY CHECK (recovery_generation > 0), + preserved_input_count INTEGER NOT NULL CHECK ( + typeof(preserved_input_count) = 'integer' AND preserved_input_count >= 0) +); +CREATE TRIGGER IF NOT EXISTS trg_history_generation_cuts_immutable +BEFORE UPDATE ON history_generation_cuts FOR EACH ROW +BEGIN SELECT RAISE(ABORT, 'history generation cuts are immutable'); END; +CREATE TRIGGER IF NOT EXISTS trg_history_generation_cuts_not_deletable +BEFORE DELETE ON history_generation_cuts FOR EACH ROW +BEGIN SELECT RAISE(ABORT, 'history generation cuts are retained throughout the era'); END; CREATE TRIGGER IF NOT EXISTS trg_history_state_single_insert BEFORE INSERT ON history_state FOR EACH ROW WHEN EXISTS (SELECT 1 FROM history_state) @@ -388,7 +401,9 @@ CREATE TRIGGER IF NOT EXISTS trg_history_generation_monotonic BEFORE UPDATE OF recovery_generation ON history_state FOR EACH ROW WHEN OLD.recovery_generation = 9223372036854775807 OR NEW.recovery_generation != OLD.recovery_generation + 1 -BEGIN SELECT RAISE(ABORT, 'recovery generation must advance by exactly one'); END; + OR NOT EXISTS (SELECT 1 FROM history_generation_cuts + WHERE recovery_generation = NEW.recovery_generation) +BEGIN SELECT RAISE(ABORT, 'recovery generation must advance by exactly one with a recorded cut'); END; CREATE TRIGGER IF NOT EXISTS trg_history_state_not_deletable BEFORE DELETE ON history_state FOR EACH ROW BEGIN SELECT RAISE(ABORT, 'history state is write-once per database'); END; diff --git a/sequencer/src/storage/mod.rs b/sequencer/src/storage/mod.rs index 9b1308d..d7fa533 100644 --- a/sequencer/src/storage/mod.rs +++ b/sequencer/src/storage/mod.rs @@ -54,7 +54,7 @@ use thiserror::Error; #[cfg(test)] pub(crate) use egress::ApplicationInputRow; -pub(crate) use egress::{HistoryReadError, L2TxContext}; +pub(crate) use egress::{HistoricalReadError, HistoryReadError, L2TxContext}; pub use history::{DirectInputExecution, HistoryState}; pub use lifecycle::{LifecycleCommand, LifecycleError, TerminalFault}; pub use open::Storage; diff --git a/sequencer/src/storage/recovery.rs b/sequencer/src/storage/recovery.rs index f09e5eb..7d8d05d 100644 --- a/sequencer/src/storage/recovery.rs +++ b/sequencer/src/storage/recovery.rs @@ -478,9 +478,9 @@ fn recover_aging_tip_inner(tx: &Transaction<'_>, danger_threshold: u64) -> Resul /// state. Accepted batch snapshots survive; before any acceptance the /// baseline supplies the rollback-safe restore point. /// 3. **Advance `RecoveryGeneration`** exactly once when the cascade -/// invalidated any valid batch. This is the externally visible statement -/// that the current era's soft-history reality changed; composing it here -/// makes generation and invalidation inseparable across crashes. +/// invalidated any valid batch, recording the surviving application count +/// before replacement directs can reuse its offsets. Cut, generation, and +/// invalidation remain inseparable across crashes. /// 4. **Reopen the Tip** the cascade just invalidated (or one a torn crash /// left missing), atomically with the cascade. Same mechanism the /// runtime's genesis path uses — see `ingress::open_fresh_tip_in_tx`. diff --git a/sequencer/src/storage/recovery_tests.rs b/sequencer/src/storage/recovery_tests.rs index a690e02..30fc268 100644 --- a/sequencer/src/storage/recovery_tests.rs +++ b/sequencer/src/storage/recovery_tests.rs @@ -11,6 +11,17 @@ use alloy_primitives::Address; use sequencer_core::l2_tx::SequencedL2Tx; use sequencer_core::protocol::ProtocolTiming; +fn generation_cuts(storage: &Storage) -> Vec<(i64, i64)> { + storage + .conn + .prepare("SELECT recovery_generation, preserved_input_count FROM history_generation_cuts ORDER BY recovery_generation") + .unwrap() + .query_map([], |row| Ok((row.get(0)?, row.get(1)?))) + .unwrap() + .collect::>() + .unwrap() +} + /// Exercise the same frame-advance-before-close sequence as the live lane. trait RecoveryFixture { fn close_batch_at( @@ -576,6 +587,11 @@ mod recover_post_flush { .expect("append safe input"); let first = storage.recover_post_flush(1200).expect("first detect"); assert_eq!(first, vec![0, 1]); + assert_eq!( + generation_cuts(&storage), + [(1, 0)], + "empty invalidated batches still record the old head" + ); assert_eq!( storage .history_state() @@ -598,6 +614,11 @@ mod recover_post_flush { let second = storage.recover_post_flush(1200).expect("second detect"); assert!(second.is_empty()); + assert_eq!( + generation_cuts(&storage), + [(1, 0)], + "a no-op must not add a cut" + ); assert_eq!( storage .history_state() @@ -1061,6 +1082,7 @@ mod tip_staleness { 0, "opening a missing Tip without invalidating history is not a recovery generation" ); + assert!(generation_cuts(&storage).is_empty()); let head = storage.open_state().expect("load open state"); assert!(head.is_some(), "recovery should have opened a fresh batch"); @@ -1210,6 +1232,10 @@ mod tip_staleness { 0, "the generation bump must roll back with the failed Tip reopen" ); + assert!( + generation_cuts(&storage).is_empty(), + "the pre-reopen cut must roll back too" + ); let invalidated_count: i64 = storage .conn .query_row( @@ -1306,6 +1332,15 @@ mod tip_staleness { .recover_post_flush(1200) .expect("detect and recover"); assert!(!invalidated.is_empty(), "should have invalidated batches"); + assert_eq!( + generation_cuts(&storage), + [(1, 0)], + "replacement directs must not enlarge the preserved prefix" + ); + assert_eq!( + storage.next_executed_input_count().unwrap(), + ExecutedInputCount::new(2) + ); let after = all_ordered_l2_txs(&mut storage); let direct_payloads: Vec<&[u8]> = after diff --git a/sequencer/src/storage/snapshot_dumps.rs b/sequencer/src/storage/snapshot_dumps.rs index de858c9..41a1b14 100644 --- a/sequencer/src/storage/snapshot_dumps.rs +++ b/sequencer/src/storage/snapshot_dumps.rs @@ -353,7 +353,7 @@ fn baseline_snapshot_in(conn: &Connection) -> Result> { .optional() } -fn finalized_dump_in(conn: &Connection) -> Result> { +pub(super) fn finalized_dump_in(conn: &Connection) -> Result> { if let Some((batch_index, nonce, inclusion_block)) = latest_accepted_boundary_in(conn)? { let snapshot = snapshot_for_batch_in(conn, batch_index)?; return Ok(Some(FinalizedDump { diff --git a/tests/e2e/Cargo.toml b/tests/e2e/Cargo.toml index 8b5845a..05c72d1 100644 --- a/tests/e2e/Cargo.toml +++ b/tests/e2e/Cargo.toml @@ -17,6 +17,8 @@ path = "src/bin/devnet_stack.rs" rollups-harness = { path = "../harness" } tracing-subscriber = { workspace = true } app-core = { path = "../../examples/app-core" } +c-app-engine = { path = "../../bindings/c-app-engine" } +c-wallet-engine = { path = "../../examples/c-wallet-engine" } sequencer-core = { path = "../../sequencer-core" } sequencer-rust-client = { path = "../../sdk/rust-client" } alloy-primitives = { workspace = true } diff --git a/tests/e2e/src/cold_replica.rs b/tests/e2e/src/cold_replica.rs index 364888b..97d4959 100644 --- a/tests/e2e/src/cold_replica.rs +++ b/tests/e2e/src/cold_replica.rs @@ -7,8 +7,11 @@ use std::time::Duration; use alloy_primitives::U256; +use app_core::application::WalletApp; +use rollups_harness::replay::apply_ws_message; use rollups_harness::{ManagedSequencer, ReplayWalletApp, TestSigner, WsClient}; use sequencer_core::api::WsTxMessage; +use sequencer_core::application::Application; use sequencer_rust_client::{ HistoryClaim, HistoryPolicyError, SequencerClient, SnapshotResponse, SubscribeError, }; @@ -17,12 +20,20 @@ use crate::ScenarioResult; use crate::test_cases::advance_live_frame_until_covers; pub(crate) async fn run(runtime: &mut ManagedSequencer) -> ScenarioResult<()> { - tokio::time::timeout(Duration::from_secs(120), run_scenario(runtime)) + run_with::(runtime).await +} + +pub(crate) async fn run_c(runtime: &mut ManagedSequencer) -> ScenarioResult<()> { + run_with::(runtime).await +} + +async fn run_with(runtime: &mut ManagedSequencer) -> ScenarioResult<()> { + tokio::time::timeout(Duration::from_secs(120), run_scenario::(runtime)) .await .map_err(|_| "cold replica scenario exceeded its 120-second deadline")? } -async fn run_scenario(runtime: &mut ManagedSequencer) -> ScenarioResult<()> { +async fn run_scenario(runtime: &mut ManagedSequencer) -> ScenarioResult<()> { let client = SequencerClient::new(runtime.endpoint())?; let genesis = client.latest_snapshot().await?; assert_eq!(genesis.claim.next_input.get(), 0); @@ -61,11 +72,11 @@ async fn run_scenario(runtime: &mut ManagedSequencer) -> ScenarioResult<()> { // after snapshot selection and before archive restoration or subscription. alice_l2.transfer(bob_address, U256::from(1_000)).await?; record(&mut reference_ws, &mut reference, &mut history).await?; - let (mut replica, downloaded_claim) = restore(snapshot).await?; + let (mut replica, downloaded_claim) = restore::(snapshot).await?; assert_eq!(downloaded_claim, original_claim); let snapshot_reference = replay_prefix(&history, snapshot_count)?; - assert_same_state(&replica, &snapshot_reference)?; - assert!(replica.executed_input_count() < reference.executed_input_count()); + assert_same_state(&mut replica, &snapshot_reference)?; + assert!(replica.executed_input_count().get() < reference.executed_input_count()); let backlog_deposit = alice_l1 .mint_and_deposit_supported_token(U256::from(70_000)) @@ -92,8 +103,8 @@ async fn run_scenario(runtime: &mut ManagedSequencer) -> ScenarioResult<()> { ScenarioResult::Ok(()) }; let consumer = async { - replica.apply(replica_ws.next_message().await?)?; - assert!(replica.executed_input_count() < backlog_head); + apply_ws_message(&mut replica, replica_ws.next_message().await?)?; + assert!(replica.executed_input_count().get() < backlog_head); first_replayed .send(()) .map_err(|_| "producer dropped the replay barrier")?; @@ -101,7 +112,7 @@ async fn run_scenario(runtime: &mut ManagedSequencer) -> ScenarioResult<()> { consume_until(&mut replica_ws, &mut replica, catch_up_target).await }; futures::try_join!(producer, consumer)?; - assert_same_state(&replica, &reference)?; + assert_same_state(&mut replica, &reference)?; replica_ws .expect_no_message_for(Duration::from_millis(100)) .await?; @@ -121,15 +132,47 @@ async fn run_scenario(runtime: &mut ManagedSequencer) -> ScenarioResult<()> { reference.executed_input_count(), ) .await?; - assert_same_state(&replica, &reference)?; + assert_same_state(&mut replica, &reference)?; assert!(replica.last_executed_safe_block() > clock_before_live); assert_eq!( - replica.current_user_balance(bob_address), + reference.current_user_balance(bob_address), U256::from(15_000) ); + // Preserve the replica's claim across a clean restart, then make the + // restarted host execute against its restored nonempty snapshot and suffix. + let restart_claim = HistoryClaim { + next_input: replica.executed_input_count(), + ..downloaded_claim + }; + drop(replica_ws); + drop(reference_ws); + runtime.stop().await?; + runtime.respawn().await?; + let client = SequencerClient::new(runtime.endpoint())?; + let mut replica_ws = WsClient::connect(&client, restart_claim).await?; + let mut reference_ws = WsClient::connect(&client, restart_claim).await?; + replica_ws + .expect_no_message_for(Duration::from_millis(100)) + .await?; + let mut alice_l2 = runtime.wallet_l2(TestSigner::from_default(1)?)?; + alice_l2.set_next_nonce(reference.current_user_nonce(alice_address)); + alice_l2.transfer(bob_address, U256::from(6_000)).await?; + record(&mut reference_ws, &mut reference, &mut history).await?; + consume_until( + &mut replica_ws, + &mut replica, + reference.executed_input_count(), + ) + .await?; + assert_same_state(&mut replica, &reference)?; + assert_eq!( + reference.current_user_balance(bob_address), + U256::from(21_000) + ); + let stale_claim = HistoryClaim { - next_input: sequencer_rust_client::ExecutedInputCount::new(replica.executed_input_count()), + next_input: replica.executed_input_count(), ..downloaded_claim }; drop(replica_ws); @@ -151,7 +194,7 @@ async fn run_scenario(runtime: &mut ManagedSequencer) -> ScenarioResult<()> { HistoryPolicyError::StaleGeneration { .. } )) )); - let (mut recovered, fresh_claim) = restore(client.latest_snapshot().await?).await?; + let (mut recovered, fresh_claim) = restore::(client.latest_snapshot().await?).await?; assert_eq!(fresh_claim.version.era_id, original_claim.version.era_id); assert_eq!( fresh_claim.version.recovery_generation.get(), @@ -162,7 +205,7 @@ async fn run_scenario(runtime: &mut ManagedSequencer) -> ScenarioResult<()> { // Reconstruct the expected replacement branch independently: the accepted // prefix survives, optimistic user ops disappear, and L1 directs replay. let mut recovered_reference = replay_prefix(&history, snapshot_count)?; - assert_same_state(&recovered, &recovered_reference)?; + assert_same_state(&mut recovered, &recovered_reference)?; for message in &history[snapshot_count as usize..] { if let WsTxMessage::DirectInput { .. } = message { let mut direct = message.clone(); @@ -179,19 +222,22 @@ async fn run_scenario(runtime: &mut ManagedSequencer) -> ScenarioResult<()> { recovered_reference.executed_input_count(), ) .await?; - assert_same_state(&recovered, &recovered_reference)?; - assert_eq!(recovered.current_user_balance(bob_address), U256::ZERO); - assert!(recovered.executed_input_count() < replica.executed_input_count()); + assert_same_state(&mut recovered, &recovered_reference)?; + assert_eq!( + recovered_reference.current_user_balance(bob_address), + U256::ZERO + ); + assert!(recovered.executed_input_count().get() < replica.executed_input_count().get()); let mut alice_l2 = runtime.wallet_l2(TestSigner::from_default(1)?)?; alice_l2.set_next_nonce(recovered_reference.current_user_nonce(alice_address)); alice_l2.transfer(bob_address, U256::from(6_000)).await?; let resumed = recovered_ws.expect_user_op_from(alice_address).await?; - recovered.apply(resumed.clone())?; + apply_ws_message(&mut recovered, resumed.clone())?; recovered_reference.apply(resumed)?; - assert_same_state(&recovered, &recovered_reference)?; + assert_same_state(&mut recovered, &recovered_reference)?; assert_eq!( - recovered.current_user_balance(bob_address), + recovered_reference.current_user_balance(bob_address), U256::from(6_000) ); recovered_ws @@ -200,7 +246,9 @@ async fn run_scenario(runtime: &mut ManagedSequencer) -> ScenarioResult<()> { Ok(()) } -async fn restore(snapshot: SnapshotResponse) -> ScenarioResult<(ReplayWalletApp, HistoryClaim)> { +pub(crate) async fn restore( + snapshot: SnapshotResponse, +) -> ScenarioResult<(A, HistoryClaim)> { let claim = snapshot.claim; assert_eq!( snapshot.response.headers()["Content-Type"], @@ -210,8 +258,8 @@ async fn restore(snapshot: SnapshotResponse) -> ScenarioResult<(ReplayWalletApp, let directory = tempfile::tempdir()?; tar::Archive::new(archive.as_ref()).unpack(directory.path())?; assert!(directory.path().join("info.toml").is_file()); - let app = ReplayWalletApp::from_dump(&directory.path().join("state"))?; - assert_eq!(app.executed_input_count(), claim.next_input.get()); + let app = A::from_dump(&directory.path().join("state"))?; + assert_eq!(app.executed_input_count(), claim.next_input); // Subsequent replay also checks that restoring does not retain a dependency // on the downloaded source directory. directory.close()?; @@ -229,15 +277,15 @@ async fn record( Ok(()) } -async fn consume_until( +async fn consume_until( ws: &mut WsClient, - app: &mut ReplayWalletApp, + app: &mut A, target: u64, ) -> ScenarioResult<()> { - while app.executed_input_count() < target { - app.apply(ws.next_message().await?)?; + while app.executed_input_count().get() < target { + apply_ws_message(app, ws.next_message().await?)?; } - assert_eq!(app.executed_input_count(), target); + assert_eq!(app.executed_input_count().get(), target); Ok(()) } @@ -249,17 +297,29 @@ fn replay_prefix(history: &[WsTxMessage], count: u64) -> ScenarioResult ScenarioResult<()> { +pub(crate) fn assert_same_state( + actual: &mut A, + expected: &ReplayWalletApp, +) -> ScenarioResult<()> { assert_eq!( - actual.executed_input_count(), + actual.executed_input_count().get(), expected.executed_input_count() ); assert_eq!( actual.last_executed_safe_block(), expected.last_executed_safe_block() ); + let progress = actual.progress(); + let directory = tempfile::tempdir()?; + let checkpoint = directory.path().join("checkpoint"); + actual.create_dump(&checkpoint)?; + assert_eq!( + actual.progress(), + progress, + "checkpoint creation preserves state" + ); assert_eq!( - actual.canonical_snapshot_bytes()?, + std::fs::read(A::state_file_in_dump(&checkpoint))?, expected.canonical_snapshot_bytes()?, "all wallet state, including balances, nonces, config, count, and clock" ); diff --git a/tests/e2e/src/lib.rs b/tests/e2e/src/lib.rs index 96ef4a2..8a9889e 100644 --- a/tests/e2e/src/lib.rs +++ b/tests/e2e/src/lib.rs @@ -5,6 +5,9 @@ mod cold_replica; pub mod test_cases; mod watchdog_compare; +// Link the reference engine's C symbols for the EngineApp replica scenarios. +use c_wallet_engine as _; + use std::future::Future; use std::pin::Pin; diff --git a/tests/e2e/src/main.rs b/tests/e2e/src/main.rs index a2c6130..2e30c8c 100644 --- a/tests/e2e/src/main.rs +++ b/tests/e2e/src/main.rs @@ -4,7 +4,7 @@ use libtest_mimic::{Arguments, Trial}; use rollups_e2e::run_trial; use rollups_harness::{ - ManagedSequencer, default_devnet_sequencer_config, devnet_sequencer_config_no_faketime, + ManagedSequencer, default_c_wallet_sequencer_config, default_devnet_sequencer_config, }; fn main() { @@ -16,20 +16,22 @@ fn main() { .map(|(name, scenario)| { Trial::test(name, move || { let log_prefix = format!("rollups-e2e-{name}"); - let spawn_config = if name == "watchdog_genesis_compare_test" - || name == "deposit_transfer_withdrawal_test" - || name == "watchdog_non_genesis_divergence_test" - { - devnet_sequencer_config_no_faketime(log_prefix) - } else if name == "fixed_fee_oracle_sets_frame_fee_test" { - let mut config = default_devnet_sequencer_config(log_prefix); - // 100 → recommended fee 1456, under the wallet client's - // DEFAULT_MAX_FEE (2500) so transfers still admit. - config.fee_oracle_fixed_log_gas_price = Some(100); - config + let mut spawn_config = if name.starts_with("c_host_") { + default_c_wallet_sequencer_config(log_prefix) } else { default_devnet_sequencer_config(log_prefix) }; + let scenario_name = name.strip_prefix("c_host_").unwrap_or(name); + if scenario_name == "watchdog_genesis_compare_test" + || scenario_name == "deposit_transfer_withdrawal_test" + || scenario_name == "watchdog_non_genesis_divergence_test" + { + spawn_config.faketime = false; + } else if scenario_name == "fixed_fee_oracle_sets_frame_fee_test" { + // 100 → recommended fee 1456, under the wallet client's + // DEFAULT_MAX_FEE (2500) so transfers still admit. + spawn_config.fee_oracle_fixed_log_gas_price = Some(100); + } run_trial(name, || async move { let mut runtime = ManagedSequencer::spawn(spawn_config).await?; let scenario_result = scenario(&mut runtime).await; diff --git a/tests/e2e/src/test_cases.rs b/tests/e2e/src/test_cases.rs index e04e86e..d2c8448 100644 --- a/tests/e2e/src/test_cases.rs +++ b/tests/e2e/src/test_cases.rs @@ -11,9 +11,10 @@ use rollups_harness::{ WsClient, sign_user_op_hex, }; use sequencer_core::api::{TxRequest, WsTxMessage}; +use sequencer_core::application::Application; use sequencer_core::fee::fee_to_linear; use sequencer_core::user_op::UserOp; -use sequencer_rust_client::SequencerClient; +use sequencer_rust_client::{HistoryPolicyError, SequencerClient, SubscribeError}; const NO_WS_MESSAGE_WAIT: Duration = Duration::from_secs(1); @@ -152,6 +153,22 @@ struct ExpectedWalletState { pub fn test_cases() -> Vec<(&'static str, ScenarioFn)> { vec![ + ( + "c_host_cold_replica_snapshot_backlog_live_recovery_test", + |runtime| Box::pin(crate::cold_replica::run_c(runtime)), + ), + ("c_host_setup_recovery_round_trip_test", |runtime| { + Box::pin(run_setup_recovery_round_trip_test::(runtime)) + }), + ("c_host_deposit_transfer_withdrawal_test", |runtime| { + Box::pin(run_deposit_transfer_withdrawal_test(runtime)) + }), + ("c_host_recovery_after_stale_batches_test", |runtime| { + Box::pin(run_recovery_after_stale_batches_test(runtime)) + }), + ("c_host_restart_and_replay_test", |runtime| { + Box::pin(run_restart_and_replay_test(runtime)) + }), ( "cold_replica_snapshot_backlog_live_recovery_test", |runtime| Box::pin(crate::cold_replica::run(runtime)), @@ -197,7 +214,9 @@ pub fn test_cases() -> Vec<(&'static str, ScenarioFn)> { Box::pin(run_recovery_after_stale_batches_test(runtime)) }), ("setup_recovery_round_trip_test", |runtime| { - Box::pin(run_setup_recovery_round_trip_test(runtime)) + Box::pin(run_setup_recovery_round_trip_test::< + app_core::application::WalletApp, + >(runtime)) }), ("sequencer_outage_pre_danger_no_recovery_test", |runtime| { Box::pin(run_sequencer_outage_pre_danger_no_recovery_test(runtime)) @@ -878,6 +897,33 @@ async fn run_restart_and_replay_test(runtime: &mut ManagedSequencer) -> Scenario replay_before_restart.last_executed_safe_block(), "mirror safe-block clock must match the pre-restart live replay clock", ); + + // Reading the persisted feed alone does not prove the restarted engine + // restored its balances and nonce. Require a fresh execution at nonce 1. + let mut resumed_alice = runtime.wallet_l2(alice)?; + resumed_alice.set_next_nonce(1); + resumed_alice.transfer(bob_address, U256::from(1)).await?; + let resumed = ws_after_restart.expect_user_op_from(alice_address).await?; + replay_after_restart.apply(resumed.clone())?; + replay_before_restart.apply(resumed)?; + assert_eq!( + replay_after_restart.canonical_snapshot_bytes()?, + replay_before_restart.canonical_snapshot_bytes()? + ); + assert_wallet_state( + &replay_after_restart, + ExpectedWalletState { + address: alice_address, + balance: expected_alice - U256::from(1) - gas, + nonce: 2, + }, + ExpectedWalletState { + address: bob_address, + balance: expected_bob + U256::from(1), + nonce: 1, + }, + 4, + ); Ok(()) } @@ -1368,7 +1414,9 @@ async fn drive_promotion_and_capture( runtime.capture_finalized_checkpoint().await } -async fn run_setup_recovery_round_trip_test(runtime: &mut ManagedSequencer) -> ScenarioResult<()> { +async fn run_setup_recovery_round_trip_test( + runtime: &mut ManagedSequencer, +) -> ScenarioResult<()> { runtime.set_max_batch_open_seconds(Some(5)); runtime.restart().await?; @@ -1399,6 +1447,10 @@ async fn run_setup_recovery_round_trip_test(runtime: &mut ManagedSequencer) -> S // Drive the batch to seal + be accepted + promoted, and capture the // resulting finalized snapshot as the recovery checkpoint. let checkpoint = drive_promotion_and_capture(runtime).await?; + let old_claim = SequencerClient::new(runtime.endpoint())? + .latest_snapshot() + .await? + .claim; eprintln!( "recovery checkpoint: B={} N={}", checkpoint.checkpoint_block, checkpoint.resume_nonce @@ -1417,19 +1469,33 @@ async fn run_setup_recovery_round_trip_test(runtime: &mut ManagedSequencer) -> S runtime.set_mine_l1_during_boot(true); runtime.respawn().await?; - // Recovery booted (the respawn above succeeded — `setup --recovery` rebuilt - // the DB and `run` started clean). Now prove the fold preserved Alice's - // logical state: a transfer at her *continuing* nonce (1) is accepted. The - // sequencer validates it against the recovered state S' — a lost nonce would - // be rejected (wrong nonce), a lost balance rejected (insufficient funds). - // So acceptance is the end-to-end proof that the checkpoint's balances + - // nonces were folded into the rebuilt DB. (The local `replay` can't verify - // S' directly — the wiped pre-recovery transfer isn't re-fed — so the - // sequencer's own acceptance is the authority here.) + // A rebuilt database cannot authorize the old replica, even when its + // numerical generation/count match. Restore and subscribe in the new era. + let client = SequencerClient::new(runtime.endpoint())?; + assert!(matches!( + client.subscribe(old_claim).await, + Err(SubscribeError::History( + HistoryPolicyError::EraChanged { .. } + )) + )); + let (mut restored, fresh_claim) = + crate::cold_replica::restore::(client.latest_snapshot().await?).await?; + assert_ne!(fresh_claim.version.era_id, old_claim.version.era_id); + assert_eq!(fresh_claim.version.recovery_generation.get(), 0); + assert!(fresh_claim.next_input.get() > 0); + crate::cold_replica::assert_same_state(&mut restored, &replay)?; + let mut resumed_ws = WsClient::connect(&client, fresh_claim).await?; + + // A continuing nonce exercises the recovered host's state as well as the + // independently restored replica and the retained reference history. let mut alice_l2_after = runtime.wallet_l2(alice)?; alice_l2_after.set_next_nonce(1); let post_transfer = U256::from(70_000_u64); alice_l2_after.transfer(bob_address, post_transfer).await?; + let message = resumed_ws.expect_user_op_from(alice_address).await?; + rollups_harness::replay::apply_ws_message(&mut restored, message.clone())?; + replay.apply(message)?; + crate::cold_replica::assert_same_state(&mut restored, &replay)?; // Explicit recovery-correctness assertions, beyond the structural // `assert_schema_invariants` (which checks `0..`-from-anchor contiguity): @@ -1458,8 +1524,7 @@ async fn run_setup_recovery_round_trip_test(runtime: &mut ManagedSequencer) -> S // from genesis — the pre-wipe batches (nonce < N') plus the post-recovery // batches at the resume nonce N' — so agreement proves the fold-rebuilt state // S' and the resumed submission both land exactly on the canonical chain - // state. The local `replay` can't check this (the wiped history is never - // re-fed), so the watchdog's independent CM is the authority. + // state. This checks native execution against the independent CM target. let floor_inclusion_block = runtime.finalized_inclusion_block().await?.unwrap_or(0); let batches_before = runtime.count_batches()?; for _ in 0..TRANSFERS_TO_FORCE_BATCH_CLOSE { diff --git a/tests/harness/src/lib.rs b/tests/harness/src/lib.rs index 53c3161..b43f1aa 100644 --- a/tests/harness/src/lib.rs +++ b/tests/harness/src/lib.rs @@ -18,8 +18,8 @@ pub use rollups::{DEVNET_CHAIN_ID, DevnetRollupsStack}; pub use sequencer::{ BatchCounts, DEFAULT_DEVNET_SEQUENCER_BIN, DEFAULT_TEST_LOGS_DIR, ManagedSequencer, ManagedSequencerConfig, RecoveryCheckpoint, RecoverySetupParams, RespawnAttemptOutcome, - RespawnPolicy, StackChildExit, default_devnet_sequencer_config, - devnet_sequencer_config_no_faketime, + RespawnPolicy, StackChildExit, default_c_wallet_sequencer_config, + default_devnet_sequencer_config, devnet_sequencer_config_no_faketime, }; pub use wallet::{ TestSigner, WalletL1Client, WalletL2Client, address_from_signing_key, sign_user_op_hex, diff --git a/tests/harness/src/paths.rs b/tests/harness/src/paths.rs index 65c8dc0..b0573f1 100644 --- a/tests/harness/src/paths.rs +++ b/tests/harness/src/paths.rs @@ -39,29 +39,35 @@ pub fn devnet_machine_image_path() -> PathBuf { workspace_root().join(DEFAULT_DEVNET_MACHINE_IMAGE_PATH) } -const DEVNET_SEQUENCER_BIN: &str = "wallet-sequencer-devnet"; - /// Resolve the `wallet-sequencer-devnet` binary built for the current Cargo invocation. -/// -/// Prefers `CARGO_TARGET_DIR` (set by `cargo run` / `cargo test` in sandboxes and -/// custom target dirs) over the workspace `target/debug/` tree, which may be stale -/// when builds only run through Cargo with a redirected target directory. pub fn resolve_devnet_sequencer_bin() -> PathBuf { - if let Ok(path) = std::env::var("CARGO_BIN_EXE_WALLET_SEQUENCER_DEVNET") { + resolve_debug_bin( + "wallet-sequencer-devnet", + "CARGO_BIN_EXE_WALLET_SEQUENCER_DEVNET", + ) +} + +pub fn resolve_c_wallet_sequencer_bin() -> PathBuf { + resolve_debug_bin("c-wallet-sequencer", "CARGO_BIN_EXE_C_WALLET_SEQUENCER") +} + +pub fn resolve_c_wallet_genesis_bin() -> PathBuf { + resolve_debug_bin("c-wallet-genesis", "CARGO_BIN_EXE_C_WALLET_GENESIS") +} + +// Prefer the active Cargo target over a potentially stale workspace target/debug. +fn resolve_debug_bin(binary: &str, override_env: &str) -> PathBuf { + if let Ok(path) = std::env::var(override_env) { let path = PathBuf::from(path); if path.exists() { return path; } } if let Ok(target) = std::env::var("CARGO_TARGET_DIR") { - let path = PathBuf::from(target) - .join("debug") - .join(DEVNET_SEQUENCER_BIN); + let path = PathBuf::from(target).join("debug").join(binary); if path.exists() { return path; } } - workspace_root() - .join("target/debug") - .join(DEVNET_SEQUENCER_BIN) + workspace_root().join("target/debug").join(binary) } diff --git a/tests/harness/src/replay.rs b/tests/harness/src/replay.rs index 4b1bf67..e715bab 100644 --- a/tests/harness/src/replay.rs +++ b/tests/harness/src/replay.rs @@ -63,10 +63,7 @@ impl ReplayWalletApp { } } -pub(crate) fn apply_ws_message( - app: &mut A, - message: WsTxMessage, -) -> HarnessResult<()> { +pub fn apply_ws_message(app: &mut A, message: WsTxMessage) -> HarnessResult<()> { let expected = app.executed_input_count().get(); if message.offset() != expected { return Err(std::io::Error::other(format!( diff --git a/tests/harness/src/sequencer.rs b/tests/harness/src/sequencer.rs index 0ab1043..cb5fc62 100644 --- a/tests/harness/src/sequencer.rs +++ b/tests/harness/src/sequencer.rs @@ -45,6 +45,9 @@ pub const DEFAULT_TEST_LOGS_DIR: &str = "tests/e2e/results"; #[derive(Debug, Clone)] pub struct ManagedSequencerConfig { pub sequencer_bin: PathBuf, + /// Optional genesis tool invoked as ` devnet` for + /// initial setup. Its output is deleted before `run` and never regenerated. + pub genesis_bin: Option, pub log_prefix: String, pub logs_dir: PathBuf, /// When false, the child runs without libfaketime (for tests that never @@ -178,6 +181,7 @@ pub struct ManagedSequencer { pub fn default_devnet_sequencer_config(log_prefix: impl Into) -> ManagedSequencerConfig { ManagedSequencerConfig { sequencer_bin: paths::resolve_devnet_sequencer_bin(), + genesis_bin: None, log_prefix: log_prefix.into(), logs_dir: PathBuf::from(DEFAULT_TEST_LOGS_DIR), faketime: true, @@ -185,6 +189,14 @@ pub fn default_devnet_sequencer_config(log_prefix: impl Into) -> Managed } } +pub fn default_c_wallet_sequencer_config(log_prefix: impl Into) -> ManagedSequencerConfig { + ManagedSequencerConfig { + sequencer_bin: paths::resolve_c_wallet_sequencer_bin(), + genesis_bin: Some(paths::resolve_c_wallet_genesis_bin()), + ..default_devnet_sequencer_config(log_prefix) + } +} + /// Devnet config without libfaketime (watchdog compare and other wall-clock-neutral tests). pub fn devnet_sequencer_config_no_faketime( log_prefix: impl Into, @@ -203,6 +215,10 @@ impl ManagedSequencer { } else { paths::resolve_from_workspace(&config.sequencer_bin) }; + let genesis_bin = config + .genesis_bin + .as_ref() + .map(paths::resolve_from_workspace); let log_prefix = config.log_prefix; let rollups = DevnetRollupsStack::spawn(log_prefix.as_str(), logs_dir.as_path()).await?; @@ -243,6 +259,7 @@ impl ManagedSequencer { // Default batch-open deadline on first boot. None, config.fee_oracle_fixed_log_gas_price, + genesis_bin.as_deref(), ) .await?; @@ -1102,6 +1119,7 @@ impl ManagedSequencer { self.recovery_setup.as_ref(), self.max_batch_open_seconds, self.fee_oracle_fixed_log_gas_price, + None, ) .await?; self.child = child; @@ -1249,6 +1267,7 @@ async fn spawn_sequencer_process( recovery: Option<&RecoverySetupParams>, max_batch_open_seconds: Option, fee_oracle_fixed_log_gas_price: Option, + genesis_bin: Option<&Path>, ) -> HarnessResult { let (endpoint, http_addr) = build_local_endpoint()?; let log_path = timestamped_log_path(logs_dir, log_prefix); @@ -1287,6 +1306,41 @@ async fn spawn_sequencer_process( let chain_id = chain_id_override.unwrap_or(DEVNET_CHAIN_ID); let bin = path_as_str(sequencer_bin)?.to_owned(); + let genesis_dir = if let Some(genesis_bin) = genesis_bin { + let dir = TempDir::new()?; + let mut command = Command::new(genesis_bin); + command + .kill_on_drop(true) + .arg(dir.path().join("state")) + .arg("devnet"); + let output = tokio::time::timeout(DEFAULT_SEQUENCER_START_TIMEOUT, command.output()) + .await + .map_err(|_| { + io_other(format!( + "genesis tool '{}' timed out", + genesis_bin.display() + )) + })? + .map_err(|err| { + io_other(format!( + "failed to run genesis tool '{}': {err}", + genesis_bin.display() + )) + })?; + if !output.status.success() { + return Err(io_other(format!( + "genesis tool '{}' failed: status={}: {}", + genesis_bin.display(), + output.status, + String::from_utf8_lossy(&output.stderr) + )) + .into()); + } + Some(dir) + } else { + None + }; + // libfaketime is applied via env vars (not the `faketime` wrapper binary), // which the file-based FAKETIME_TIMESTAMP_FILE mechanism reads on every // time call (FAKETIME_NO_CACHE=1) so tests can shift the clock at runtime. @@ -1305,6 +1359,10 @@ async fn spawn_sequencer_process( // (e.g. a chain-id mismatch) surfaces here as a non-zero exit, just as it // surfaced from the monolithic boot before the split. let mut setup_cmd = Command::new(&bin); + setup_cmd.env_remove("CARTESI_SEQUENCER_STATE_FILE"); + if let Some(dir) = &genesis_dir { + setup_cmd.arg("--state-file").arg(dir.path().join("state")); + } if let (Some(lib), Some(rc)) = (libfaketime_path, faketime_rc_path) { apply_faketime_env(&mut setup_cmd, lib, rc)?; } @@ -1396,10 +1454,16 @@ async fn spawn_sequencer_process( )) .into()); } + if let Some(dir) = genesis_dir { + // Neither startup nor later recovery may depend on the original source. + dir.close() + .map_err(|err| io_other(format!("remove initial genesis after setup: {err}")))?; + } // Phase B — `run` (re-spawned on every restart; reads identity from the // setup DB). let mut run_cmd = Command::new(&bin); + run_cmd.env_remove("CARTESI_SEQUENCER_STATE_FILE"); if let (Some(lib), Some(rc)) = (libfaketime_path, faketime_rc_path) { apply_faketime_env(&mut run_cmd, lib, rc)?; }