From e092b48b7d9cfa06913503fdbf96026d0c6ceee9 Mon Sep 17 00:00:00 2001 From: gcdepaula Date: Wed, 9 Sep 2026 15:46:54 -0300 Subject: [PATCH 01/29] feat(ffi): add native application bridge and conformance tests Adapt Eduardo Bart's C application support from PR #32 to the revised Application contract. Keep exclusive movable engine ownership, native progress, fallible validation/execution, and typed missing/corrupt checkpoint errors. Add a reference wallet engine, host binaries, and tests through the actual ABI. Source: https://github.com/cartesi/sequencer/pull/32 Source-Commit: 0fa1755a882ce8aeeb6ba7877ba4ea7c479da9d1 --- Cargo.lock | 42 ++ Cargo.toml | 4 + examples/c-app-engine/Cargo.toml | 19 + examples/c-app-engine/README.md | 64 +++ examples/c-app-engine/build.rs | 159 ++++++ .../c-app-engine/include/application-engine.h | 402 ++++++++++++++++ examples/c-app-engine/src/lib.rs | 288 +++++++++++ examples/c-app-engine/src/sys.rs | 20 + examples/c-app-sequencer/Cargo.toml | 22 + examples/c-app-sequencer/build.rs | 15 + examples/c-app-sequencer/src/lib.rs | 72 +++ examples/c-app-sequencer/src/main.rs | 24 + examples/c-wallet-engine/Cargo.toml | 30 ++ .../src/bin/c-wallet-genesis.rs | 34 ++ examples/c-wallet-engine/src/lib.rs | 454 ++++++++++++++++++ examples/c-wallet-engine/tests/conformance.rs | 237 +++++++++ examples/c-wallet-sequencer/Cargo.toml | 15 + examples/c-wallet-sequencer/src/main.rs | 16 + 18 files changed, 1917 insertions(+) create mode 100644 examples/c-app-engine/Cargo.toml create mode 100644 examples/c-app-engine/README.md create mode 100644 examples/c-app-engine/build.rs create mode 100644 examples/c-app-engine/include/application-engine.h create mode 100644 examples/c-app-engine/src/lib.rs create mode 100644 examples/c-app-engine/src/sys.rs create mode 100644 examples/c-app-sequencer/Cargo.toml create mode 100644 examples/c-app-sequencer/build.rs create mode 100644 examples/c-app-sequencer/src/lib.rs create mode 100644 examples/c-app-sequencer/src/main.rs create mode 100644 examples/c-wallet-engine/Cargo.toml create mode 100644 examples/c-wallet-engine/src/bin/c-wallet-genesis.rs create mode 100644 examples/c-wallet-engine/src/lib.rs create mode 100644 examples/c-wallet-engine/tests/conformance.rs create mode 100644 examples/c-wallet-sequencer/Cargo.toml create mode 100644 examples/c-wallet-sequencer/src/main.rs diff --git a/Cargo.lock b/Cargo.lock index 942aacf4..bdac5421 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1374,6 +1374,27 @@ dependencies = [ "serde", ] +[[package]] +name = "c-app-engine" +version = "0.1.0" +dependencies = [ + "alloy-primitives", + "bindgen", + "sequencer-core", +] + +[[package]] +name = "c-app-sequencer" +version = "0.1.0" +dependencies = [ + "c-app-engine", + "clap", + "sequencer", + "tokio", + "tracing", + "tracing-subscriber", +] + [[package]] name = "c-kzg" version = "2.1.8" @@ -1389,6 +1410,27 @@ dependencies = [ "serde", ] +[[package]] +name = "c-wallet-engine" +version = "0.1.0" +dependencies = [ + "alloy-primitives", + "app-core", + "c-app-engine", + "ethereum_ssz", + "sequencer-core", + "tempfile", +] + +[[package]] +name = "c-wallet-sequencer" +version = "0.1.0" +dependencies = [ + "c-app-sequencer", + "c-wallet-engine", + "tokio", +] + [[package]] name = "canonical-app" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index f2622793..67a19a09 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,6 +8,10 @@ members = [ "examples/canonical-app", "examples/canonical-test", "examples/wallet-sequencer", + "examples/c-app-engine", + "examples/c-app-sequencer", + "examples/c-wallet-engine", + "examples/c-wallet-sequencer", "tests/benchmarks", "tests/harness", "tests/e2e", diff --git a/examples/c-app-engine/Cargo.toml b/examples/c-app-engine/Cargo.toml new file mode 100644 index 00000000..98f6443a --- /dev/null +++ b/examples/c-app-engine/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "c-app-engine" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "Application shim over a static engine library implementing the application-engine C API" +homepage.workspace = true +repository.workspace = true +readme = "README.md" +authors.workspace = true + +[build-dependencies] +# Generates the FFI declarations from the engine header at build time. It decides how this host +# reads every record crossing the seam, so a bump changes the ABI interpretation. +bindgen = "0.72" + +[dependencies] +sequencer-core = { path = "../../sequencer-core" } +alloy-primitives = { workspace = true } diff --git a/examples/c-app-engine/README.md b/examples/c-app-engine/README.md new file mode 100644 index 00000000..89b72296 --- /dev/null +++ b/examples/c-app-engine/README.md @@ -0,0 +1,64 @@ +# C application bridge + +`EngineApp` implements `sequencer_core::application::Application` using the +[application-engine C ABI](include/application-engine.h). The sequencer owns one +engine at a time. A handle can move between threads; calls on it never overlap. +The bridge is `Send`, without `Clone` or `Sync`. + +The native engine owns application state and its execution count/safe-block +clock. Successful execution advances that progress, including counted no-ops; +protocol rejection leaves it unchanged. The shared Rust execution functions +check these transitions. Fatal validation, execution, or output-drain failures +return `AppError`; callers must discard that instance. Exceptions must not cross +the C ABI. + +`NOT_FOUND` and `INVALID_DUMP` retain the distinction between missing/corrupt +checkpoint artifacts and other operational `IO_ERROR` failures. Error strings +are diagnostics and never determine classification. The engine and generated +bindings must use the same header, including these status declarations. + +A dump prefix may be a file or a directory. Opening it produces independently +mutable state without changing the source; checkpoint creation may mutate the +engine's backing arrangement, while preserving logical state and progress. +Successful checkpoints are durable and immutable under subsequent execution. +`state_file_in_dump` names the one canonical comparison file, which can be the +whole dump or a projection alongside richer restoration artifacts. `EngineApp` +does not implement the optional Rust `CanonicalState` inspection trait. + +## Reference wallet + +The reference engine exports the Rust wallet through actual `extern "C"` +functions. Its static archive is also usable by the generic host. In the +repository's development shell: + +```sh +cargo run -p c-wallet-engine --bin c-wallet-genesis -- /tmp/wallet-genesis devnet +cargo run -p c-wallet-sequencer -- --state-file /tmp/wallet-genesis setup +cargo run -p c-wallet-sequencer -- run +cargo test -p c-wallet-engine --test conformance +``` + +The ordinary setup/run environment configuration is still required. The genesis +path is required only for plain `setup`; warm startup, `flush-mempool`, and +`setup --recovery` use the sequencer's durable checkpoints. + +## External engine + +Build the application's static archive and use the corresponding header: + +```sh +APPLICATION_ENGINE_LIB=/absolute/path/libengine.a \ +APPLICATION_ENGINE_HEADER=/absolute/path/application-engine.h \ +APPLICATION_ENGINE_METHOD_PAYLOAD_LIMIT=53 \ + cargo build -p c-app-sequencer +``` + +The payload limit must match the engine's own build. Bindgen generates the Rust +records from that header, so a build needs libclang. The engine also supplies its +own genesis tool; configuration does not cross this ABI. With no external +archive configured, the generic binary reports that no engine was linked, and +`c-wallet-sequencer` supplies the reference implementation through Cargo. + +The conformance suite compares native and ABI execution over mixed inputs, +notices and vouchers, rejection/no-op progress, dump round trips, independent +instances, and fatal/error classification. diff --git a/examples/c-app-engine/build.rs b/examples/c-app-engine/build.rs new file mode 100644 index 00000000..af7f13ee --- /dev/null +++ b/examples/c-app-engine/build.rs @@ -0,0 +1,159 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +//! Links the application's engine archive and generates the FFI declarations from its header. +//! +//! The three environment variables are the whole application-specific binding, documented in +//! `README.md`. With none of them set this crate links no archive, +//! and the binary that uses it supplies the engine instead. + +use std::env; +use std::path::{Path, PathBuf}; + +/// The in-workspace wallet engine's own bound, used when a build declares none. +/// +/// Only reachable when `c-wallet-engine` is the engine, which asserts this same value against +/// `WalletApp::MAX_METHOD_PAYLOAD_BYTES`, so a number that drifts fails that crate's compile +/// rather than reaching a host. An application outside this workspace always declares its own. +const REFERENCE_ENGINE_METHOD_PAYLOAD_LIMIT: u32 = 1 + 32 + 20; + +/// A ceiling on what an application may declare, since the bound gates ingress. +const MAX_METHOD_PAYLOAD_LIMIT: u32 = 1 << 20; + +/// Generate `sys`'s contents from the engine header, the one authoritative declaration of what +/// the archive exports, so a change on the engine side is either picked up here or fails this +/// build. +/// +/// The payload bound is defined for the parse rather than read out of the header, because the +/// header deliberately refuses to carry a default for it. +fn generate_bindings(header: &Path, method_payload_limit: u32) { + let out_path = PathBuf::from(env::var("OUT_DIR").expect("OUT_DIR")).join("bindings.rs"); + let bindings = bindgen::Builder::default() + .header( + header + .to_str() + .expect("APPLICATION_ENGINE_HEADER is not valid UTF-8"), + ) + // Parse the C arm of the header. Its C++ arm only spells noexcept, which has no bearing + // on the ABI and no Rust spelling. + .clang_args(["-x", "c", "-std=c11"]) + .clang_arg(format!( + "-DAPPLICATION_ENGINE_METHOD_PAYLOAD_LIMIT={method_payload_limit}" + )) + // Only the seam's own surface, never what stdint.h drags in behind it + .allowlist_item("^(application_engine_|ApplicationEngine|APPLICATION_ENGINE_).*") + // Plain integer constants, never Rust enums. The contract requires refusing a value the + // engine added later, which holding it in a Rust enum would make undefined behavior. + .default_enum_style(bindgen::EnumVariation::Consts) + // The C constants already carry the APPLICATION_ENGINE_ prefix, so repeating the enum's + // name in front of them would spell them differently here than in the header + .prepend_enum_name(false) + .rust_edition(bindgen::RustEdition::Edition2024) + .generate() + .expect("failed to generate bindings from APPLICATION_ENGINE_HEADER"); + bindings + .write_to_file(&out_path) + .unwrap_or_else(|err| panic!("failed to write {}: {err}", out_path.display())); +} + +/// Link the application's own archive, the path a deployment outside this workspace takes. +/// +/// The link name is the archive's own name, the linker has no other way to spell it. +fn link_application_archive(engine_lib: &Path) { + // Fail loudly at build time instead of at link time with a confusing message + assert!( + engine_lib.is_file(), + "APPLICATION_ENGINE_LIB points at {}, which is not a file, build the engine archive first", + engine_lib.display() + ); + + let file_name = engine_lib + .file_name() + .and_then(|name| name.to_str()) + .expect("APPLICATION_ENGINE_LIB is not valid UTF-8"); + let link_name = file_name + .strip_prefix("lib") + .and_then(|name| name.strip_suffix(".a")) + .unwrap_or_else(|| { + panic!("APPLICATION_ENGINE_LIB names {file_name}, expected a static library named lib.a") + }); + // A bare file name has an empty parent, search the working directory then + let engine_lib_dir = engine_lib + .parent() + .filter(|dir| !dir.as_os_str().is_empty()) + .unwrap_or(Path::new(".")); + + println!("cargo::rerun-if-changed={}", engine_lib.display()); + println!( + "cargo::rustc-link-search=native={}", + engine_lib_dir.display() + ); + println!("cargo::rustc-link-lib=static={link_name}"); + + // Engines are commonly implemented in C or C++, and one that needs no C++ runtime links this + // harmlessly + let cxx_runtime = match env::var("CARGO_CFG_TARGET_OS").expect("target os").as_str() { + "macos" => "c++", + _ => "stdc++", + }; + println!("cargo::rustc-link-lib={cxx_runtime}"); +} + +/// What an application supplying its own archive has to declare alongside it. +/// +/// Both are demanded rather than defaulted. The archive and the header are separate artifacts and +/// only that pairing is meaningful, and a bound guessed here would be exactly the silently wrong +/// number the header's own `#error` exists to prevent. +fn external_engine(engine_lib: &str) -> (PathBuf, u32) { + link_application_archive(Path::new(engine_lib)); + + let header = PathBuf::from(env::var("APPLICATION_ENGINE_HEADER").expect( + "APPLICATION_ENGINE_HEADER is unset, point it at the header the archive was built against", + )); + assert!( + header.is_file(), + "APPLICATION_ENGINE_HEADER points at {}, which is not a file", + header.display() + ); + + let declared = env::var("APPLICATION_ENGINE_METHOD_PAYLOAD_LIMIT").expect( + "APPLICATION_ENGINE_METHOD_PAYLOAD_LIMIT is unset, set it to the application's largest \ + method payload, the same value the archive was built with", + ); + let limit = declared.trim().parse::().unwrap_or_else(|err| { + panic!("APPLICATION_ENGINE_METHOD_PAYLOAD_LIMIT is not a number: {err}") + }); + // The bound gates ingress and sizes batches, so a fat-fingered value is worth refusing here + // rather than discovering as a memory bill + assert!( + limit > 0 && limit <= MAX_METHOD_PAYLOAD_LIMIT, + "APPLICATION_ENGINE_METHOD_PAYLOAD_LIMIT is {limit}, expected 1..={MAX_METHOD_PAYLOAD_LIMIT}" + ); + (header, limit) +} + +fn main() { + println!("cargo::rerun-if-env-changed=APPLICATION_ENGINE_LIB"); + println!("cargo::rerun-if-env-changed=APPLICATION_ENGINE_HEADER"); + println!("cargo::rerun-if-env-changed=APPLICATION_ENGINE_METHOD_PAYLOAD_LIMIT"); + + let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR")); + let (header, method_payload_limit) = match env::var("APPLICATION_ENGINE_LIB") { + Ok(engine_lib) => external_engine(&engine_lib), + // Linked from inside this workspace, where `c-wallet-engine` is the engine + Err(_) => { + assert!( + env::var_os("APPLICATION_ENGINE_HEADER").is_none() + && env::var_os("APPLICATION_ENGINE_METHOD_PAYLOAD_LIMIT").is_none(), + "external engine settings require APPLICATION_ENGINE_LIB" + ); + ( + manifest_dir.join("include").join("application-engine.h"), + REFERENCE_ENGINE_METHOD_PAYLOAD_LIMIT, + ) + } + }; + + println!("cargo::rerun-if-changed={}", header.display()); + generate_bindings(&header, method_payload_limit); +} diff --git a/examples/c-app-engine/include/application-engine.h b/examples/c-app-engine/include/application-engine.h new file mode 100644 index 00000000..6fea11ad --- /dev/null +++ b/examples/c-app-engine/include/application-engine.h @@ -0,0 +1,402 @@ +/* (c) Cartesi and individual authors (see AUTHORS) */ +/* SPDX-License-Identifier: Apache-2.0 (see LICENSE) */ + +#ifndef APPLICATION_ENGINE_H +#define APPLICATION_ENGINE_H + +#include + +/// @file +/// The C API of an application engine, and the only surface an engine exports. It mirrors the +/// Cartesi sequencer's Application contract, so the host owning the fold stays application +/// agnostic and an engine is swappable behind this header. Plain C so any host can consume it. +/// +/// An application implements these declarations into a static archive, which the `c-app-engine` +/// shim links at build time to become the sequencer's Application, and which the application's +/// own canonical binary links natively from the same objects. One compiled engine on both sides +/// is what makes off-chain and on-chain execution deterministic. An application written in C++, +/// or in any language with a C ABI, implements it the same way. +/// +/// Nothing application specific crosses. An engine is handed a dump already holding a configured +/// deployment, so a host never learns what configures the application it runs, and the path is +/// opaque, a file or a directory as the engine chooses. +/// +/// This header is the surface a host binds to, and the Rust host generates its declarations from +/// it with bindgen rather than restating them, so a signature changed here cannot disagree with +/// the host that links the engine. The records that cross are plain C layout and the +/// engine must static_assert their sizes and field offsets, so a compiler laying one out +/// differently fails its build rather than the seam. A generated binding carries the same checks +/// on the host side. +/// +/// A generated binding follows whatever this header says, so the header carries the compatibility +/// obligation on its own. Every vocabulary here is append only and no value is ever reused, since +/// a host is entitled to refuse a value it does not know rather than to have it renumbered +/// underneath. +/// +/// Widths are fixed, nothing crosses as size_t. A C enum's underlying type is implementation +/// defined, so an engine must static_assert each one's width against int32_t, which is what lets +/// a host mirror them as a plain 32-bit integer. +/// +/// Boundary rules, binding on every function here. Every entry point is total over the bytes it +/// is handed: a payload arrives from whoever signed or posted it, so a method an engine cannot +/// parse is a refusal or a counted no-op, never INTERNAL_ERROR. Reporting INTERNAL_ERROR there +/// would hand any caller a process kill, since it is fatal-no-resume. +/// +/// No exception may cross. Fallible entry points report errors through status codes; a fatal +/// validation or execution error defines no successor and the caller must discard the instance. +/// Lifecycle statuses distinguish operational I/O from missing or malformed dump artifacts. +/// Only accept or reject is consensus visible; rejection diagnostics are descriptive. +/// +/// Errors are errno style. A fallible entry point returns a status (or a null pointer for +/// application_engine_state_file_in_dump) and leaves the reason for +/// application_engine_get_last_error_message. Statuses are the contract, messages are diagnostics, +/// never branch on their text. + +/// @brief Marks the public C API, exported even when the consumer builds with +/// -fvisibility=hidden. Carried by declarations only, definitions inherit it. +#if defined(__GNUC__) || defined(__clang__) +#define APPLICATION_ENGINE_API __attribute__((visibility("default"))) +#else +#define APPLICATION_ENGINE_API +#endif + +/// @brief Spells the non-throwing guarantee for a C++ consumer, C has no equivalent. +/// @details Part of every signature, a throw reaching the seam terminates rather than unwinding +/// into a caller that has no way to handle it. +#ifdef __cplusplus +#define APPLICATION_ENGINE_NOEXCEPT noexcept +#else +#define APPLICATION_ENGINE_NOEXCEPT +#endif + +/// @brief Size in bytes of an account address crossing the seam. +/// @details An engine must static_assert it against its own address type, so the two cannot +/// drift. +#define APPLICATION_ENGINE_ADDRESS_SIZE 20 + +/// @brief Size in bytes of an on-chain amount crossing the seam. +/// @details An amount is an EVM word, wider than any scalar this API carries, so it crosses as +/// raw big-endian bytes rather than as a number a host may not be able to spell. +#define APPLICATION_ENGINE_VALUE_SIZE 32 + +/// @brief The one value this header does not fix, supplied by the application's build. +/// @details The ingress bound on a single user op's method payload is an application sizing +/// decision, the largest payload any of its methods can carry, so it is defined on the compile +/// line rather than here. Every consumer of this header, the engine's own translation units and +/// the binding generation alike, must be given the same value, which is what keeps the bound the +/// host enforces and the bound the engine parses under from being two numbers. +/// +/// There is deliberately no default. A silently wrong bound is the exact failure this +/// declaration exists to prevent, so an undefined one stops the build here. +#ifndef APPLICATION_ENGINE_METHOD_PAYLOAD_LIMIT +#error "define APPLICATION_ENGINE_METHOD_PAYLOAD_LIMIT to the application's largest method payload" +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +/// @brief The bounds an engine declares, spelled as constants a generated binding can read. +/// @details A binding generator sees a macro only where it is defined, and +/// APPLICATION_ENGINE_METHOD_PAYLOAD_LIMIT is defined on the compile line, so the value is +/// restated here as an enumeration constant. That is what carries it across to a host: the +/// application defines one number, and both sides read it from this declaration. +typedef enum ApplicationEngineLimits { + /// The largest method payload a user op may carry, in bytes. The host publishes it as the + /// sequencer's MAX_METHOD_PAYLOAD_BYTES and refuses anything larger, so an engine that raised + /// its own bound without raising this one would never see the payloads it grew to accept. + APPLICATION_ENGINE_MAX_METHOD_PAYLOAD_BYTES = APPLICATION_ENGINE_METHOD_PAYLOAD_LIMIT, +} ApplicationEngineLimits; + +/// @brief An account address, raw bytes and no encoding. +/// @details A named type rather than a loose buffer, so an address and an amount cannot be +/// passed for one another. +typedef struct ApplicationEngineEthereumAddress { + uint8_t bytes[APPLICATION_ENGINE_ADDRESS_SIZE]; ///< The address bytes, in on-chain order. +} ApplicationEngineEthereumAddress; + +/// @brief A 256-bit unsigned amount, big-endian. +/// @details The width every on-chain amount crosses at, whatever the engine prices it in +/// internally, so no amount is narrowed to fit through here. +typedef struct ApplicationEngineUint256 { + uint8_t bytes[APPLICATION_ENGINE_VALUE_SIZE]; ///< The amount bytes, most significant first. +} ApplicationEngineUint256; + +/// @brief A borrowed range of bytes, never owned by the side receiving it. +/// @details Const because nothing crossing here is written through, inputs and output payloads +/// alike are read only for whoever receives them. +typedef struct ApplicationEngineByteSpan { + const uint8_t *data; ///< The first byte, null only when the range is empty. + uint64_t size; ///< The number of bytes. +} ApplicationEngineByteSpan; + +/// @brief Status returned by every fallible entry point. Zero is success and every failure is +/// negative, so `status < 0` is the failure test and a value added later never disturbs it. +/// @details Append only. The status is the contract, the accompanying message is a diagnostic. +typedef enum ApplicationEngineStatus { + APPLICATION_ENGINE_STATUS_OK = 0, ///< Call succeeded. + APPLICATION_ENGINE_STATUS_INVALID = -1, ///< Refused by protocol or configuration validation. + APPLICATION_ENGINE_STATUS_INTERNAL_ERROR = -2, ///< Internal engine failure, fatal-no-resume. + APPLICATION_ENGINE_STATUS_IO_ERROR = -3, ///< An operational filesystem or mapping failure. + APPLICATION_ENGINE_STATUS_NOT_FOUND = -4, ///< A required dump artifact is missing. + APPLICATION_ENGINE_STATUS_INVALID_DUMP = -5, ///< A dump is malformed or truncated. +} ApplicationEngineStatus; + +/// @brief A user op as its sender signed it. +/// @details The engine reads the nonce and the data and carries max_fee without checking it, +/// that guard belongs to the caller, so an op arrives whole. +typedef struct ApplicationEngineUserOp { + uint32_t nonce; ///< Sender replay protection nonce. + uint16_t max_fee; ///< Highest frame fee price the sender accepts, in log space. + ApplicationEngineByteSpan data; ///< Method payload, opaque here and parsed by the engine. +} ApplicationEngineUserOp; + +/// @brief A user op that already passed validation, as the caller sequenced it. +/// @details Not the signed op: execution consumes the nonce the state expects, and the max fee +/// went with the guard the caller already settled, leaving the fee the frame charges. +typedef struct ApplicationEngineValidUserOp { + ApplicationEngineEthereumAddress sender; ///< The recovered signer. + uint16_t fee; ///< The frame fee price charged, in log space. + ApplicationEngineByteSpan data; ///< Method payload, opaque here and parsed by the engine. +} ApplicationEngineValidUserOp; + +/// @brief An input taken straight from the L1 input box. +/// @details Its sender is authenticated by the chain rather than recovered from a signature, +/// which is what lets the engine trust it without validating anything first. +typedef struct ApplicationEngineDirectInput { + ApplicationEngineEthereumAddress sender; ///< The L1 authenticated sender. + uint64_t block_number; ///< The L1 inclusion block number. + ApplicationEngineByteSpan payload; ///< The raw input payload. +} ApplicationEngineDirectInput; + +/// @brief Why an engine refused a user op, mirroring the sequencer's rejection reasons. +/// @details Append only. Written only on APPLICATION_ENGINE_STATUS_INVALID, and it selects which +/// member of ApplicationEngineInvalidValues carries the diagnostics. +typedef enum ApplicationEngineInvalidReason { + APPLICATION_ENGINE_INVALID_NONCE = 0, ///< Nonce or account binding, read `nonce`. + APPLICATION_ENGINE_INVALID_MAX_FEE = 1, ///< The caller-owned max fee guard, read `max_fee`. + APPLICATION_ENGINE_INSUFFICIENT_FEE_BALANCE = 2, ///< Cannot cover the frame fee, read `fee_balance`. +} ApplicationEngineInvalidReason; + +/// @brief Diagnostics for APPLICATION_ENGINE_INVALID_NONCE. +typedef struct ApplicationEngineInvalidNonce { + uint32_t expected; ///< The nonce the account expects next. + uint32_t got; ///< The nonce the op carried. +} ApplicationEngineInvalidNonce; + +/// @brief Diagnostics for APPLICATION_ENGINE_INVALID_MAX_FEE. +/// @details Both values are log space exponents, base 129/128. No engine produces this reason, +/// the guard belongs to the caller, it is carried so the reason vocabulary stays whole. +typedef struct ApplicationEngineInvalidMaxFee { + uint16_t max_fee; ///< The highest frame fee price the sender accepts. + uint16_t base_fee; ///< The frame fee price charged. +} ApplicationEngineInvalidMaxFee; + +/// @brief Diagnostics for APPLICATION_ENGINE_INSUFFICIENT_FEE_BALANCE. +/// @details Both amounts are in the smallest unit of whatever the engine charges fees in. An +/// all-ones required means a fee no balance could ever cover, an engine reports it that way +/// rather than reporting an amount it cannot represent. +typedef struct ApplicationEngineInsufficientFeeBalance { + ApplicationEngineUint256 required; ///< The frame fee the sender must cover. + ApplicationEngineUint256 available; ///< What the sender has free to cover it. +} ApplicationEngineInsufficientFeeBalance; + +/// @brief The diagnostics of a refusal, read through the member its reason selects. +typedef union ApplicationEngineInvalidValues { + ApplicationEngineInvalidNonce nonce; ///< APPLICATION_ENGINE_INVALID_NONCE. + ApplicationEngineInvalidMaxFee max_fee; ///< APPLICATION_ENGINE_INVALID_MAX_FEE. + ApplicationEngineInsufficientFeeBalance fee_balance; ///< APPLICATION_ENGINE_INSUFFICIENT_FEE_BALANCE. +} ApplicationEngineInvalidValues; + +/// @brief A refusal, its reason and the diagnostics that reason selects. +/// @details Written whole and only on APPLICATION_ENGINE_STATUS_INVALID. Reading a member other +/// than the one the reason names is a caller bug, the unselected members hold nothing. +typedef struct ApplicationEngineInvalid { + ApplicationEngineInvalidReason reason; ///< Why the op was refused. + ApplicationEngineInvalidValues values; ///< The diagnostics for that reason. +} ApplicationEngineInvalid; + +/// @brief The kinds of output an engine can emit, mirroring the rollup output types. +/// @details Append only. A host that meets a kind it does not know has drifted from the engine +/// and must refuse rather than guess, the kinds carry different payload meanings. +typedef enum ApplicationEngineOutputKind { + APPLICATION_ENGINE_OUTPUT_VOUCHER = 0, ///< A call to a destination, read `voucher`. + APPLICATION_ENGINE_OUTPUT_NOTICE = 1, ///< A payload only attestation, read `notice`. +} ApplicationEngineOutputKind; + +/// @brief A call the chain makes on the application's behalf. +typedef struct ApplicationEngineVoucher { + ApplicationEngineEthereumAddress destination; ///< The contract to call. + ApplicationEngineUint256 value; ///< The call value in wei. + ApplicationEngineByteSpan payload; ///< The encoded call payload. +} ApplicationEngineVoucher; + +/// @brief An output's body, read through the member its kind selects. +typedef union ApplicationEngineOutputValues { + ApplicationEngineVoucher voucher; ///< APPLICATION_ENGINE_OUTPUT_VOUCHER. + ApplicationEngineByteSpan notice; ///< APPLICATION_ENGINE_OUTPUT_NOTICE, the attested payload. +} ApplicationEngineOutputValues; + +/// @brief An output an execution emitted, its kind and the body that kind selects. +/// @details Written whole and only when a drain returns OK. Reading a member other than the one +/// the kind names is a caller bug, the unselected members hold nothing. +typedef struct ApplicationEngineOutput { + ApplicationEngineOutputKind kind; ///< What the engine emitted. + ApplicationEngineOutputValues values; ///< The body for that kind. +} ApplicationEngineOutput; + +/// @brief The engine instance behind the handle, opaque to every caller. +typedef struct ApplicationEngine ApplicationEngine; + +/// @brief Get the message describing the most recent failure. +/// @returns A NUL terminated string, never null, empty when the last fallible call succeeded. +/// @details Read it after a negative status or a null handle. Every fallible entry point clears +/// it on entry, so it always describes the call that just failed, and the next one overwrites +/// it, so copy rather than retain the pointer. Infallible entry points leave it untouched. +/// The storage is thread local, matching the ownership the seam already requires: an engine may +/// move between threads but its handle is never used by two at once. +/// +/// The entry points that take no handle are the exception and must be reentrant. A host may call +/// application_engine_state_file_in_dump from request handlers while an execution is in flight, +/// so an engine answering out of one process-wide buffer would race. This message and that path +/// are both thread local here for that reason. +APPLICATION_ENGINE_API const char *application_engine_get_last_error_message(void) APPLICATION_ENGINE_NOEXCEPT; + +/// @brief Open an engine over a dump holding an existing deployment. +/// @param prefix The dump to open, carrying whatever shape the engine gives a dump. +/// @param out_engine The engine handle, written only on OK and left untouched otherwise. +/// @returns OK, NOT_FOUND for a missing artifact, INVALID_DUMP for malformed/truncated bytes, +/// IO_ERROR for other filesystem failures, or INTERNAL_ERROR for an engine invariant failure. +/// @details The only way to open an engine, and it never creates. A state is written once at +/// genesis by a tool that knows the application's configuration, and every engine afterwards opens +/// what is there, which is what keeps deployment configuration off this API. +/// +/// Loading must leave the source dump immutable. Two engines may open the same dump and evolve +/// independently, and deleting the source after loading must leave both usable. Private mapping, +/// deserialization, or an independently owned working copy can implement this contract. The caller +/// never mutates a source dump underneath a live engine. +/// +/// The deployment found there is validated before the handle is written, the backstop for a state +/// that was truncated, hand edited, or left by a genesis that died mid-write. +APPLICATION_ENGINE_API ApplicationEngineStatus application_engine_from_dump(const char *prefix, + ApplicationEngine **out_engine) APPLICATION_ENGINE_NOEXCEPT; + +/// @brief Destroy an engine instance. +/// @param engine The engine handle. +APPLICATION_ENGINE_API void application_engine_destroy(ApplicationEngine *engine) APPLICATION_ENGINE_NOEXCEPT; + +/// @brief Validate a user op against current state, pure and read-only. +/// @param engine The engine handle. +/// @param sender The recovered signer. +/// @param user_op The op to validate, as its sender signed it. +/// @param current_fee The frame fee price in log space. +/// @param out_invalid Why the op was refused, written whole and only on INVALID. +/// @returns OK, INVALID with diagnostics, or INTERNAL_ERROR. +/// @details A rejection reports itself through out_invalid and leaves the last error message +/// empty, only INTERNAL_ERROR carries one. The max-fee guard belongs to the caller and is never +/// checked here, so APPLICATION_ENGINE_INVALID_MAX_FEE never comes back from this call. Queued +/// outputs are left alone, only an execution touches them. +APPLICATION_ENGINE_API ApplicationEngineStatus application_engine_validate_user_op(const ApplicationEngine *engine, + const ApplicationEngineEthereumAddress *sender, const ApplicationEngineUserOp *user_op, uint16_t current_fee, + ApplicationEngineInvalid *out_invalid) APPLICATION_ENGINE_NOEXCEPT; + +/// @brief Execute a validated user op, consuming the current expected nonce. +/// @param engine The engine handle. +/// @param user_op The validated op to execute. +/// @param safe_block The covering frame safe block, folded into the clock as max(clock, it). +/// @param out_output_count How many outputs this op left waiting, written only on OK. +/// @returns OK or INTERNAL_ERROR (an engine throw is fatal-no-resume). +/// @details An op the method rejects still executed and still counts, so it reports OK. Only +/// accept or reject is consensus visible and the state carries it, the seam does not surface +/// the application's own reason. +/// +/// An execution refuses to run while an earlier execution's outputs are still queued, reporting +/// INTERNAL_ERROR without executing anything rather than discarding outputs meant to reach the +/// chain. That refusal is what makes the count reported this op's own. +APPLICATION_ENGINE_API ApplicationEngineStatus application_engine_execute_valid_user_op(ApplicationEngine *engine, + const ApplicationEngineValidUserOp *user_op, uint64_t safe_block, + uint64_t *out_output_count) APPLICATION_ENGINE_NOEXCEPT; + +/// @brief Execute a direct input from the L1 input box. +/// @param engine The engine handle. +/// @param input The input to execute, its L1 block folded into the clock as max(clock, it). +/// @param out_output_count How many outputs this input left waiting, written only on OK. +/// @returns OK or INTERNAL_ERROR (an engine throw is fatal-no-resume). +/// @details An input the engine rejects is a counted no-op and still reports OK, the same way a +/// rejected user op does. Outputs behave as they do for a user op. +APPLICATION_ENGINE_API ApplicationEngineStatus application_engine_execute_direct_input(ApplicationEngine *engine, + const ApplicationEngineDirectInput *input, uint64_t *out_output_count) APPLICATION_ENGINE_NOEXCEPT; + +/// @brief Take the next queued output, in emission order. +/// @param engine The engine handle. +/// @param out_output The output taken, written whole and only on OK. +/// @returns OK with an output written, or INTERNAL_ERROR. +/// @details Call it exactly as many times as the execution reported, which is what attributes +/// the outputs to the input that produced them. Taking one more than were queued is a caller bug +/// and reports INTERNAL_ERROR rather than an empty output a host might act on. The payload +/// pointer stays valid until the next drain call releases it, so copy before draining again. +/// A voucher carries its value even when an engine only ever emits zero-value ones, so a host +/// reads what the engine emitted instead of assuming it. +APPLICATION_ENGINE_API ApplicationEngineStatus application_engine_drain_output(ApplicationEngine *engine, + ApplicationEngineOutput *out_output) APPLICATION_ENGINE_NOEXCEPT; + +/// @brief Get the maximum block carried by any executed input (the engine's safe-block clock). +/// @param engine The engine handle. +/// @returns The last executed safe block, zero when nothing has executed. +/// @details Carried by execution rather than set, so an engine cannot execute and forget to +/// advance it. It lives in the state, so a resumed one reports the block it reflects. +APPLICATION_ENGINE_API uint64_t application_engine_last_executed_safe_block( + const ApplicationEngine *engine) APPLICATION_ENGINE_NOEXCEPT; + +/// @brief Get the count of executed inputs, user ops and direct inputs alike. +/// @param engine The engine handle. +/// @returns The executed input count. +APPLICATION_ENGINE_API uint64_t application_engine_executed_input_count( + const ApplicationEngine *engine) APPLICATION_ENGINE_NOEXCEPT; + +/// @brief Create a crash durable dump of the engine state (write, fsync). +/// @param engine The engine handle. +/// @param prefix The dump to create, must not pre-exist. It carries whatever shape the engine's +/// state does, a directory or a plain file as the engine chooses. +/// @returns OK, IO_ERROR when the filesystem refused, which is what a full filesystem or an +/// exhausted quota reports, or INTERNAL_ERROR. +/// @details Must be called at a quiescent point only, no in-flight execution. On OK the dump +/// survives an immediate kernel crash, its payload and the directory entry naming it are both +/// synchronized before returning. An engine that cleans up after a failed write leaves the prefix +/// free for a clean retry, which a host cannot do on its behalf. +/// +/// The dump contains the current state. The engine may change backing files or reopen internal +/// handles while checkpointing, but application state and progress stay unchanged. Subsequent +/// execution must leave the completed dump immutable. +APPLICATION_ENGINE_API ApplicationEngineStatus application_engine_create_dump(ApplicationEngine *engine, + const char *prefix) APPLICATION_ENGINE_NOEXCEPT; + +/// @brief Delete a previously created dump. +/// @param prefix The dump to remove. +/// @returns OK, NOT_FOUND when the dump is absent, IO_ERROR for other filesystem failures, +/// or INTERNAL_ERROR. +/// @details An engine still holding this dump open keeps running, its mapping outlives the name. +/// Synchronizing the directory entry before returning is what would let a caller drop its record +/// of the path on OK, so an engine that skips it can leave an orphan behind a crash. +APPLICATION_ENGINE_API ApplicationEngineStatus application_engine_delete_dump( + const char *prefix) APPLICATION_ENGINE_NOEXCEPT; + +/// @brief Get the path of the canonical state file inside a dump. +/// @param prefix The dump to name the state file of. +/// @returns A NUL terminated path, or null on failure with the reason in the last error message. +/// @details Pure over the prefix, it touches no filesystem and needs no engine. Where the state +/// file sits follows from the shape the engine gives a dump, which is why the engine answers +/// rather than a host assuming. An engine whose dump is a directory answers with a file inside +/// it, and one whose dump is the state image itself answers with the prefix unchanged. +/// +/// The storage is engine owned and thread local, overwritten by the next call on the same +/// thread, so copy rather than retain the pointer. Being fallible, it also clears the last error +/// message like any other fallible call, so read a failure's message before asking this. +APPLICATION_ENGINE_API const char *application_engine_state_file_in_dump( + const char *prefix) APPLICATION_ENGINE_NOEXCEPT; + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#endif /* APPLICATION_ENGINE_H */ diff --git a/examples/c-app-engine/src/lib.rs b/examples/c-app-engine/src/lib.rs new file mode 100644 index 00000000..b2edc38d --- /dev/null +++ b/examples/c-app-engine/src/lib.rs @@ -0,0 +1,288 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +//! An exclusively owned application engine behind the C ABI in `application-engine.h`. + +pub mod sys; + +use std::ffi::{CStr, CString, OsStr}; +use std::os::unix::ffi::OsStrExt; +use std::path::{Path, PathBuf}; + +use alloy_primitives::{Address, U256}; +use sequencer_core::application::{ + AppError, AppOutput, AppOutputs, ApplicationProgress, InvalidReason, ValidationOutcome, +}; +use sequencer_core::history::ExecutedInputCount; +use sequencer_core::l2_tx::{DirectInput, ValidUserOp}; +use sequencer_core::user_op::UserOp; + +pub use sequencer_core::application::Application; + +fn path_to_cstring(path: &Path) -> CString { + CString::new(path.as_os_str().as_bytes()) + .unwrap_or_else(|_| panic!("path contains an interior NUL: {}", path.display())) +} + +fn abi_address(address: Address) -> sys::ApplicationEngineEthereumAddress { + sys::ApplicationEngineEthereumAddress { + bytes: address.into_array(), + } +} + +fn abi_span(bytes: &[u8]) -> sys::ApplicationEngineByteSpan { + sys::ApplicationEngineByteSpan { + data: bytes.as_ptr(), + size: u64::try_from(bytes.len()).expect("length exceeds the ABI's u64 width"), + } +} + +// The engine retains each payload until its next drain call, so copy before calling again. +fn payload_from(span: sys::ApplicationEngineByteSpan) -> Vec { + if span.size == 0 { + return Vec::new(); + } + assert!( + !span.data.is_null(), + "non-empty output payload has a null pointer" + ); + let len = usize::try_from(span.size).expect("output length exceeds usize"); + unsafe { std::slice::from_raw_parts(span.data, len) }.to_vec() +} + +fn last_error_message() -> String { + let message = unsafe { sys::application_engine_get_last_error_message() }; + assert!(!message.is_null(), "engine returned a null error message"); + unsafe { CStr::from_ptr(message) } + .to_string_lossy() + .into_owned() +} + +fn internal(reason: impl Into) -> AppError { + AppError::Internal { + reason: reason.into(), + } +} + +fn check(status: i32, operation: &str) -> Result<(), AppError> { + if status == sys::APPLICATION_ENGINE_STATUS_OK { + return Ok(()); + } + let reason = format!( + "engine {operation} failed (status {status}): {}", + last_error_message() + ); + let kind = match status { + sys::APPLICATION_ENGINE_STATUS_IO_ERROR => std::io::ErrorKind::Other, + sys::APPLICATION_ENGINE_STATUS_NOT_FOUND => std::io::ErrorKind::NotFound, + sys::APPLICATION_ENGINE_STATUS_INVALID_DUMP => std::io::ErrorKind::InvalidData, + _ => return Err(internal(reason)), + }; + Err(AppError::Io(std::io::Error::new(kind, reason))) +} + +/// Owns one engine handle. Calls on that handle never overlap. +pub struct EngineApp { + engine: *mut sys::ApplicationEngine, +} + +// SAFETY: the ABI permits moving a handle between threads. Ownership is exclusive; +// EngineApp is neither Clone nor Sync, and mutation requires &mut self. +unsafe impl Send for EngineApp {} + +impl Drop for EngineApp { + fn drop(&mut self) { + unsafe { sys::application_engine_destroy(self.engine) }; + } +} + +impl EngineApp { + fn drain_outputs(&mut self, count: u64) -> Result { + let mut outputs = AppOutputs::new(); + for _ in 0..count { + let mut output = sys::ApplicationEngineOutput { + kind: 0, + values: sys::ApplicationEngineOutputValues { + notice: abi_span(&[]), + }, + }; + check( + unsafe { sys::application_engine_drain_output(self.engine, &mut output) }, + "drain_output", + )?; + outputs.push(match output.kind { + sys::APPLICATION_ENGINE_OUTPUT_VOUCHER => { + let voucher = unsafe { output.values.voucher }; + AppOutput::Voucher { + destination: Address::from(voucher.destination.bytes), + value: U256::from_be_bytes(voucher.value.bytes), + payload: payload_from(voucher.payload), + } + } + sys::APPLICATION_ENGINE_OUTPUT_NOTICE => { + AppOutput::Notice(payload_from(unsafe { output.values.notice })) + } + other => { + return Err(internal(format!( + "engine reported unknown output kind {other}" + ))); + } + }); + } + Ok(outputs) + } +} + +impl Application for EngineApp { + const MAX_METHOD_PAYLOAD_BYTES: usize = + sys::APPLICATION_ENGINE_MAX_METHOD_PAYLOAD_BYTES as usize; + + fn validate_user_op( + &self, + sender: Address, + user_op: &UserOp, + current_fee: u16, + ) -> Result { + let mut invalid = sys::ApplicationEngineInvalid { + reason: 0, + values: sys::ApplicationEngineInvalidValues { + nonce: sys::ApplicationEngineInvalidNonce { + expected: 0, + got: 0, + }, + }, + }; + let op = sys::ApplicationEngineUserOp { + nonce: user_op.nonce, + max_fee: user_op.max_fee, + data: abi_span(user_op.data.as_ref()), + }; + let status = unsafe { + sys::application_engine_validate_user_op( + self.engine, + &abi_address(sender), + &op, + current_fee, + &mut invalid, + ) + }; + if status != sys::APPLICATION_ENGINE_STATUS_INVALID { + check(status, "validate_user_op")?; + return Ok(ValidationOutcome::Accept); + } + let reason = match invalid.reason { + sys::APPLICATION_ENGINE_INVALID_NONCE => { + let nonce = unsafe { invalid.values.nonce }; + InvalidReason::InvalidNonce { + expected: nonce.expected, + got: nonce.got, + } + } + sys::APPLICATION_ENGINE_INSUFFICIENT_FEE_BALANCE => { + let balance = unsafe { invalid.values.fee_balance }; + InvalidReason::InsufficientFeeBalance { + required: U256::from_be_bytes(balance.required.bytes), + available: U256::from_be_bytes(balance.available.bytes), + } + } + // Max-fee rejection belongs to the shared execution boundary. + other => { + return Err(internal(format!( + "engine reported unsupported invalid reason {other}" + ))); + } + }; + Ok(ValidationOutcome::Reject(reason)) + } + + fn apply_valid_user_op( + &mut self, + user_op: &ValidUserOp, + safe_block: u64, + ) -> Result { + let op = sys::ApplicationEngineValidUserOp { + sender: abi_address(user_op.sender), + fee: user_op.fee, + data: abi_span(&user_op.data), + }; + let mut count = 0; + check( + unsafe { + sys::application_engine_execute_valid_user_op( + self.engine, + &op, + safe_block, + &mut count, + ) + }, + "execute_valid_user_op", + )?; + self.drain_outputs(count) + } + + fn apply_direct_input(&mut self, input: &DirectInput) -> Result { + let direct = sys::ApplicationEngineDirectInput { + sender: abi_address(input.sender), + block_number: input.block_number, + payload: abi_span(&input.payload), + }; + let mut count = 0; + check( + unsafe { + sys::application_engine_execute_direct_input(self.engine, &direct, &mut count) + }, + "execute_direct_input", + )?; + self.drain_outputs(count) + } + + fn progress(&self) -> ApplicationProgress { + let count = unsafe { sys::application_engine_executed_input_count(self.engine) }; + let clock = unsafe { sys::application_engine_last_executed_safe_block(self.engine) }; + ApplicationProgress::try_new(ExecutedInputCount::new(count), clock) + .expect("engine returned incoherent application progress") + } + + fn from_dump(prefix: &Path) -> Result { + let prefix = path_to_cstring(prefix); + let mut engine = std::ptr::null_mut(); + check( + unsafe { sys::application_engine_from_dump(prefix.as_ptr(), &mut engine) }, + "from_dump", + )?; + assert!( + !engine.is_null(), + "engine reported successful load without a handle" + ); + Ok(Self { engine }) + } + + fn create_dump(&mut self, prefix: &Path) -> Result<(), AppError> { + let prefix = path_to_cstring(prefix); + check( + unsafe { sys::application_engine_create_dump(self.engine, prefix.as_ptr()) }, + "create_dump", + ) + } + + fn delete_dump(prefix: &Path) -> Result<(), AppError> { + let prefix = path_to_cstring(prefix); + check( + unsafe { sys::application_engine_delete_dump(prefix.as_ptr()) }, + "delete_dump", + ) + } + + fn state_file_in_dump(prefix: &Path) -> PathBuf { + let prefix = path_to_cstring(prefix); + let state_file = unsafe { sys::application_engine_state_file_in_dump(prefix.as_ptr()) }; + assert!( + !state_file.is_null(), + "engine could not name its state file: {}", + last_error_message() + ); + PathBuf::from(OsStr::from_bytes( + unsafe { CStr::from_ptr(state_file) }.to_bytes(), + )) + } +} diff --git a/examples/c-app-engine/src/sys.rs b/examples/c-app-engine/src/sys.rs new file mode 100644 index 00000000..87325e0d --- /dev/null +++ b/examples/c-app-engine/src/sys.rs @@ -0,0 +1,20 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +//! The engine C ABI, generated by bindgen from the header. Nothing here is written by hand, so +//! to change what crosses, change the header. +//! +//! Enums arrive as plain integer constants, never Rust enums, so a value an engine adds later is +//! a number this host can refuse rather than undefined behavior. Records carry generated layout +//! assertions. + +// Generated code follows C's naming and carries the whole surface, including what this host has +// no call for yet +#![allow( + non_upper_case_globals, + non_camel_case_types, + non_snake_case, + dead_code +)] + +include!(concat!(env!("OUT_DIR"), "/bindings.rs")); diff --git a/examples/c-app-sequencer/Cargo.toml b/examples/c-app-sequencer/Cargo.toml new file mode 100644 index 00000000..0b04b800 --- /dev/null +++ b/examples/c-app-sequencer/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "c-app-sequencer" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "Sequencer host for a C application engine linked as a static library" +homepage.workspace = true +repository.workspace = true +readme = "../../README.md" +authors.workspace = true + +[[bin]] +name = "c-app-sequencer" +path = "src/main.rs" + +[dependencies] +c-app-engine = { path = "../c-app-engine" } +sequencer = { path = "../../sequencer" } +clap = { workspace = true, features = ["env"] } +tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } +tracing = { workspace = true } +tracing-subscriber = { workspace = true } diff --git a/examples/c-app-sequencer/build.rs b/examples/c-app-sequencer/build.rs new file mode 100644 index 00000000..bf4320c2 --- /dev/null +++ b/examples/c-app-sequencer/build.rs @@ -0,0 +1,15 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +//! Tells the generic binary whether an engine archive was supplied. +//! +//! A cargo feature would be the usual way, but features are additive and `--all-features` would +//! turn it on in builds with no archive, which is exactly the combination that cannot link. + +fn main() { + println!("cargo::rustc-check-cfg=cfg(external_engine)"); + println!("cargo::rerun-if-env-changed=APPLICATION_ENGINE_LIB"); + if std::env::var_os("APPLICATION_ENGINE_LIB").is_some() { + println!("cargo::rustc-cfg=external_engine"); + } +} diff --git a/examples/c-app-sequencer/src/lib.rs b/examples/c-app-sequencer/src/lib.rs new file mode 100644 index 00000000..32c69f45 --- /dev/null +++ b/examples/c-app-sequencer/src/lib.rs @@ -0,0 +1,72 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +//! Host wiring for a C application: the sequencer library over `c-app-engine`'s shim. +//! +//! An application's binary crate is the few lines in `c-wallet-sequencer`, the same shape +//! `wallet-sequencer` has for a Rust application: link an engine, call [`run`]. + +use std::io::IsTerminal; +use std::path::PathBuf; +use std::process::ExitCode; + +use c_app_engine::{Application, EngineApp}; +use clap::Parser; +use tracing_subscriber::EnvFilter; + +/// The sequencer library's subcommands plus the one option the host owns, the engine state. +#[derive(Debug, Parser)] +#[command( + version, + about = "Rollup sequencer host for a C application.\n\n\ + Runs the application engine linked in at build time, the one implementing the \ + application-engine C API. The subcommands come from the sequencer library.\n\n\ + All options can also be set via environment variables (shown in brackets)." +)] +struct Cli { + /// Engine genesis state, read-only and load-bearing for `setup` alone, `run` opens dumps. + /// Must already hold a deployment written by the application's genesis tool + #[arg(long, env = "CARTESI_SEQUENCER_STATE_FILE", value_name = "PATH")] + state_file: Option, + #[command(subcommand)] + command: sequencer::Command, +} + +/// Parse this host's arguments and run the sequencer over the linked engine. +pub async fn run() -> ExitCode { + // Parse first so `--help`/`--version` work without the engine state + let Cli { + state_file, + command, + } = Cli::parse(); + + tracing_subscriber::fmt() + .with_ansi(std::io::stdout().is_terminal()) + .with_env_filter( + EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info")), + ) + .init(); + + // Only `setup` starts from this file, `run` and `flush-mempool` work from the dumps the + // sequencer took, so demanding it of them would keep a warm deployment from restarting once + // the genesis state is gone. Opened here rather than inside the closure so a state the engine + // cannot read names what to do about it instead of raising the closure's panic. + let mut app = None; + if matches!(&command, sequencer::Command::Setup(config) if !config.recovery) { + let Some(state_file) = state_file else { + tracing::error!("plain setup requires --state-file or CARTESI_SEQUENCER_STATE_FILE"); + return ExitCode::FAILURE; + }; + match EngineApp::from_dump(&state_file) { + Ok(engine) => app = Some(engine), + Err(err) => { + tracing::error!(state = %state_file.display(), + "cannot open the engine state, write one with the application's genesis tool first: {err:?}"); + return ExitCode::FAILURE; + } + } + } + + // Only plain setup invokes the genesis constructor. + sequencer::dispatch(command, move || app.expect("engine opened for setup")).await +} diff --git a/examples/c-app-sequencer/src/main.rs b/examples/c-app-sequencer/src/main.rs new file mode 100644 index 00000000..ce2a970e --- /dev/null +++ b/examples/c-app-sequencer/src/main.rs @@ -0,0 +1,24 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +//! The generic host binary for an application supplying its engine as an archive, which needs no +//! code of its own. Built without one it has no engine to run and says so; nothing in the host is +//! reachable from that arm, which is what lets the workspace build it with no symbols to resolve. + +use std::process::ExitCode; + +#[cfg(external_engine)] +#[tokio::main] +async fn main() -> ExitCode { + c_app_sequencer::run().await +} + +#[cfg(not(external_engine))] +fn main() -> ExitCode { + eprintln!( + "built without an engine, so there is no application to run. Set \ + APPLICATION_ENGINE_LIB, APPLICATION_ENGINE_HEADER and \ + APPLICATION_ENGINE_METHOD_PAYLOAD_LIMIT, then build again." + ); + ExitCode::FAILURE +} diff --git a/examples/c-wallet-engine/Cargo.toml b/examples/c-wallet-engine/Cargo.toml new file mode 100644 index 00000000..6099778c --- /dev/null +++ b/examples/c-wallet-engine/Cargo.toml @@ -0,0 +1,30 @@ +[package] +name = "c-wallet-engine" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "The placeholder wallet app exported as an application-engine C API static library" +homepage.workspace = true +repository.workspace = true +readme = "../../README.md" +authors.workspace = true + +[lib] +# `staticlib` is the artifact a C application's build would consume, and it is what proves this +# crate really does export the C API. `rlib` is what an in-workspace binary links instead, so +# `cargo build` resolves the symbols itself and needs no archive path handed to it. +crate-type = ["staticlib", "rlib"] + +[[bin]] +name = "c-wallet-genesis" +path = "src/bin/c-wallet-genesis.rs" + +[dependencies] +app-core = { path = "../app-core" } +c-app-engine = { path = "../c-app-engine" } +sequencer-core = { path = "../../sequencer-core" } +alloy-primitives = { workspace = true } + +[dev-dependencies] +tempfile = { workspace = true } +ssz = { workspace = true } diff --git a/examples/c-wallet-engine/src/bin/c-wallet-genesis.rs b/examples/c-wallet-engine/src/bin/c-wallet-genesis.rs new file mode 100644 index 00000000..ddb63090 --- /dev/null +++ b/examples/c-wallet-engine/src/bin/c-wallet-genesis.rs @@ -0,0 +1,34 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +//! Writes a genesis state for the wallet engine. The seam has no create path, so every +//! application ships a tool like this, and the host never learns what it configured. + +use std::path::PathBuf; +use std::process::ExitCode; + +use app_core::application::WalletConfig; + +fn main() -> ExitCode { + let arguments: Vec = std::env::args().skip(1).collect(); + let config = match arguments.as_slice() { + [_, preset] if preset == "devnet" => WalletConfig::devnet(), + [_, preset] if preset == "sepolia" => WalletConfig::sepolia(), + [_] => WalletConfig::default(), + _ => { + eprintln!( + "usage: c-wallet-genesis [devnet|sepolia]\n\n\ + Writes a genesis wallet state at , which must not already exist." + ); + return ExitCode::from(2); + } + }; + + let state_dir = PathBuf::from(&arguments[0]); + if let Err(err) = c_wallet_engine::write_genesis(&state_dir, config) { + eprintln!("cannot write {}: {err:?}", state_dir.display()); + return ExitCode::FAILURE; + } + println!("wrote {}", state_dir.display()); + ExitCode::SUCCESS +} diff --git a/examples/c-wallet-engine/src/lib.rs b/examples/c-wallet-engine/src/lib.rs new file mode 100644 index 00000000..56af2367 --- /dev/null +++ b/examples/c-wallet-engine/src/lib.rs @@ -0,0 +1,454 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +//! `app-core`'s wallet exported through the application-engine C API, building to +//! `libc_wallet_engine.a`. The rules it implements are stated in that header. +//! +//! Records come from `c-app-engine::sys`, generated from the same header, so producer and +//! consumer read one declaration. A `panic!` here aborts, which is the policy the header wants. + +use std::cell::RefCell; +use std::collections::VecDeque; +use std::ffi::{CString, OsStr, c_char}; +use std::os::unix::ffi::OsStrExt; +use std::path::{Path, PathBuf}; + +use alloy_primitives::Address; +use app_core::application::{WalletApp, WalletConfig}; +use c_app_engine::sys; +use sequencer_core::application::{ + AppError, AppOutput, AppOutputs, Application, InvalidReason, ValidationOutcome, +}; +use sequencer_core::l2_tx::{DirectInput, ValidUserOp}; +use sequencer_core::user_op::UserOp; + +// The header carries no default for the ingress bound, so a build supplies it. This is what +// makes the two agree: a build that told the host a different number than the wallet implements +// fails here rather than at the boundary. +const _: () = assert!( + sys::APPLICATION_ENGINE_MAX_METHOD_PAYLOAD_BYTES as usize + == WalletApp::MAX_METHOD_PAYLOAD_BYTES, + "the payload bound this build declares to the host is not the wallet's own" +); + +thread_local! { + /// The last failure's message, and the buffer `state_file_in_dump` answers out of. + /// + /// Thread local because the header requires the handle-free entry points to be reentrant. + static LAST_ERROR: RefCell = RefCell::new(CString::default()); + static STATE_FILE: RefCell = RefCell::new(CString::default()); +} + +fn clear_error() { + LAST_ERROR.with(|slot| *slot.borrow_mut() = CString::default()); +} + +fn set_error(message: impl AsRef) { + // A NUL inside a diagnostic is not worth failing over, truncate at it + let message = message.as_ref(); + let bytes = message.split('\0').next().unwrap_or_default().as_bytes(); + LAST_ERROR.with(|slot| { + *slot.borrow_mut() = CString::new(bytes).unwrap_or_default(); + }); +} + +/// Preserve missing/corrupt artifact classification across the ABI; diagnostics are not parsed. +fn report(error: &AppError, what: &str) -> sys::ApplicationEngineStatus { + match error { + AppError::Io(err) => { + set_error(format!("{what} failed: {err}")); + match err.kind() { + std::io::ErrorKind::NotFound => sys::APPLICATION_ENGINE_STATUS_NOT_FOUND, + std::io::ErrorKind::InvalidData + | std::io::ErrorKind::UnexpectedEof + | std::io::ErrorKind::NotADirectory + | std::io::ErrorKind::IsADirectory => sys::APPLICATION_ENGINE_STATUS_INVALID_DUMP, + _ => sys::APPLICATION_ENGINE_STATUS_IO_ERROR, + } + } + other => { + set_error(format!("{what} failed: {other:?}")); + sys::APPLICATION_ENGINE_STATUS_INTERNAL_ERROR + } + } +} + +/// Borrow a path the way the C API carries it, raw bytes rather than text. +/// +/// A Unix path is bytes. Decoding it as UTF-8 would answer for a different path than the caller +/// named whenever it is not valid UTF-8. +/// +/// # Safety +/// `path` must be a non-null NUL terminated string that outlives the call. +unsafe fn path_from(path: *const c_char) -> PathBuf { + assert!(!path.is_null(), "the C API forbids a null path"); + let bytes = unsafe { std::ffi::CStr::from_ptr(path) }.to_bytes(); + PathBuf::from(OsStr::from_bytes(bytes)) +} + +/// Borrow a byte span the way the C API carries it. +/// +/// # Safety +/// The span must describe a readable range that outlives the call, or be empty. +unsafe fn slice_from<'a>(span: &sys::ApplicationEngineByteSpan) -> &'a [u8] { + if span.size == 0 { + // An empty span may carry a null pointer, which Rust slices reject even when empty + return &[]; + } + assert!(!span.data.is_null(), "non-empty span with a null pointer"); + let len = usize::try_from(span.size).expect("span length exceeds usize on this host"); + unsafe { std::slice::from_raw_parts(span.data, len) } +} + +/// The engine instance behind the opaque handle. +pub struct ApplicationEngine { + app: WalletApp, + /// What the last execution produced, in emission order, still to be drained. + pending: VecDeque, + /// The payload the last drain handed out. Held here because the span the caller receives + /// borrows it, and the contract keeps it alive until the next drain releases it. + drained_payload: Vec, +} + +/// Take a handle the C API was given. +/// +/// # Safety +/// `engine` must be a live handle from `application_engine_from_dump`, not yet destroyed. +unsafe fn engine_ref<'a>(engine: *const ApplicationEngine) -> &'a ApplicationEngine { + assert!(!engine.is_null(), "the C API forbids a null engine handle"); + unsafe { &*engine } +} + +/// # Safety +/// As [`engine_ref`], and no other reference to the engine may be live. +unsafe fn engine_mut<'a>(engine: *mut ApplicationEngine) -> &'a mut ApplicationEngine { + assert!(!engine.is_null(), "the C API forbids a null engine handle"); + unsafe { &mut *engine } +} + +#[unsafe(no_mangle)] +pub extern "C" fn application_engine_get_last_error_message() -> *const c_char { + LAST_ERROR.with(|slot| slot.borrow().as_ptr()) +} + +/// # Safety +/// `prefix` is a NUL terminated path and `out_engine` is writable. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn application_engine_from_dump( + prefix: *const c_char, + out_engine: *mut *mut ApplicationEngine, +) -> sys::ApplicationEngineStatus { + clear_error(); + let prefix = unsafe { path_from(prefix) }; + match WalletApp::from_dump(&prefix) { + Ok(app) => { + let engine = Box::new(ApplicationEngine { + app, + pending: VecDeque::new(), + drained_payload: Vec::new(), + }); + unsafe { *out_engine = Box::into_raw(engine) }; + sys::APPLICATION_ENGINE_STATUS_OK + } + // WalletApp::from_dump only reads and decodes; its Internal errors identify malformed bytes. + Err(AppError::Internal { reason }) => { + set_error(format!("from_dump failed: {reason}")); + sys::APPLICATION_ENGINE_STATUS_INVALID_DUMP + } + Err(err) => report(&err, "from_dump"), + } +} + +/// # Safety +/// `engine` is a live handle, and it is not used again afterwards. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn application_engine_destroy(engine: *mut ApplicationEngine) { + if engine.is_null() { + return; + } + drop(unsafe { Box::from_raw(engine) }); +} + +/// # Safety +/// Every pointer is non-null and its pointee outlives the call. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn application_engine_validate_user_op( + engine: *const ApplicationEngine, + sender: *const sys::ApplicationEngineEthereumAddress, + user_op: *const sys::ApplicationEngineUserOp, + current_fee: u16, + out_invalid: *mut sys::ApplicationEngineInvalid, +) -> sys::ApplicationEngineStatus { + clear_error(); + let engine = unsafe { engine_ref(engine) }; + let sender = Address::from(unsafe { (*sender).bytes }); + let user_op = unsafe { &*user_op }; + let op = UserOp { + nonce: user_op.nonce, + max_fee: user_op.max_fee, + data: unsafe { slice_from(&user_op.data) }.to_vec().into(), + }; + + match engine.app.validate_user_op(sender, &op, current_fee) { + Ok(ValidationOutcome::Accept) => sys::APPLICATION_ENGINE_STATUS_OK, + Err(err) => report(&err, "validate_user_op"), + Ok(ValidationOutcome::Reject(reason)) => { + let invalid = match reason { + InvalidReason::InvalidNonce { expected, got } => sys::ApplicationEngineInvalid { + reason: sys::APPLICATION_ENGINE_INVALID_NONCE, + values: sys::ApplicationEngineInvalidValues { + nonce: sys::ApplicationEngineInvalidNonce { expected, got }, + }, + }, + InvalidReason::InsufficientFeeBalance { + required, + available, + } => sys::ApplicationEngineInvalid { + reason: sys::APPLICATION_ENGINE_INSUFFICIENT_FEE_BALANCE, + values: sys::ApplicationEngineInvalidValues { + fee_balance: sys::ApplicationEngineInsufficientFeeBalance { + required: sys::ApplicationEngineUint256 { + bytes: required.to_be_bytes(), + }, + available: sys::ApplicationEngineUint256 { + bytes: available.to_be_bytes(), + }, + }, + }, + }, + // The caller owns the max-fee guard and this entry point never checks it, so the + // app cannot produce this reason. Reporting it would be a lie about which union + // member carries the diagnostics. + InvalidReason::InvalidMaxFee { .. } => { + set_error("the app reported a caller-owned max fee rejection"); + return sys::APPLICATION_ENGINE_STATUS_INTERNAL_ERROR; + } + }; + unsafe { *out_invalid = invalid }; + sys::APPLICATION_ENGINE_STATUS_INVALID + } + } +} + +/// Take an execution's outputs, refusing to run over ones still queued. +/// +/// The refusal is what makes the count an execution reports its own. Discarding them instead +/// would drop outputs bound for the chain. +/// # Safety +/// `out_output_count` is writable and outlives the call. +unsafe fn execute( + engine: &mut ApplicationEngine, + out_output_count: *mut u64, + run: impl FnOnce(&mut WalletApp) -> Result, +) -> sys::ApplicationEngineStatus { + if !engine.pending.is_empty() { + set_error("an earlier execution's outputs are still queued"); + return sys::APPLICATION_ENGINE_STATUS_INTERNAL_ERROR; + } + match run(&mut engine.app) { + Ok(outputs) => { + engine.pending = outputs.into(); + // SAFETY: the caller guarantees the pointer is writable. + unsafe { *out_output_count = engine.pending.len() as u64 }; + sys::APPLICATION_ENGINE_STATUS_OK + } + // Application errors define no successor; the caller must discard this instance. + Err(err) => report(&err, "execute"), + } +} + +/// # Safety +/// Every pointer is non-null and its pointee outlives the call. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn application_engine_execute_valid_user_op( + engine: *mut ApplicationEngine, + user_op: *const sys::ApplicationEngineValidUserOp, + safe_block: u64, + out_output_count: *mut u64, +) -> sys::ApplicationEngineStatus { + clear_error(); + let engine = unsafe { engine_mut(engine) }; + let user_op = unsafe { &*user_op }; + let op = ValidUserOp { + sender: Address::from(user_op.sender.bytes), + fee: user_op.fee, + data: unsafe { slice_from(&user_op.data) }.to_vec(), + }; + // SAFETY: the caller guarantees `out_output_count` is writable. + unsafe { + execute(engine, out_output_count, |app| { + app.apply_valid_user_op(&op, safe_block) + }) + } +} + +/// # Safety +/// Every pointer is non-null and its pointee outlives the call. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn application_engine_execute_direct_input( + engine: *mut ApplicationEngine, + input: *const sys::ApplicationEngineDirectInput, + out_output_count: *mut u64, +) -> sys::ApplicationEngineStatus { + clear_error(); + let engine = unsafe { engine_mut(engine) }; + let input = unsafe { &*input }; + let direct = DirectInput { + sender: Address::from(input.sender.bytes), + block_number: input.block_number, + payload: unsafe { slice_from(&input.payload) }.to_vec(), + }; + // SAFETY: the caller guarantees `out_output_count` is writable. + unsafe { + execute(engine, out_output_count, |app| { + app.apply_direct_input(&direct) + }) + } +} + +/// # Safety +/// `engine` is a live handle and `out_output` is writable. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn application_engine_drain_output( + engine: *mut ApplicationEngine, + out_output: *mut sys::ApplicationEngineOutput, +) -> sys::ApplicationEngineStatus { + clear_error(); + let engine = unsafe { engine_mut(engine) }; + // Taking one more than were queued is a caller bug, reported rather than answered with an + // empty output a host might act on + let Some(output) = engine.pending.pop_front() else { + set_error("no output is queued"); + return sys::APPLICATION_ENGINE_STATUS_INTERNAL_ERROR; + }; + + let written = match output { + AppOutput::Voucher { + destination, + value, + payload, + } => { + engine.drained_payload = payload; + sys::ApplicationEngineOutput { + kind: sys::APPLICATION_ENGINE_OUTPUT_VOUCHER, + values: sys::ApplicationEngineOutputValues { + voucher: sys::ApplicationEngineVoucher { + destination: sys::ApplicationEngineEthereumAddress { + bytes: destination.into_array(), + }, + value: sys::ApplicationEngineUint256 { + bytes: value.to_be_bytes(), + }, + payload: span_of(&engine.drained_payload), + }, + }, + } + } + AppOutput::Notice(payload) => { + engine.drained_payload = payload; + sys::ApplicationEngineOutput { + kind: sys::APPLICATION_ENGINE_OUTPUT_NOTICE, + values: sys::ApplicationEngineOutputValues { + notice: span_of(&engine.drained_payload), + }, + } + } + }; + unsafe { *out_output = written }; + sys::APPLICATION_ENGINE_STATUS_OK +} + +/// Lend a payload to the caller. It stays valid until the next drain replaces it. +fn span_of(payload: &[u8]) -> sys::ApplicationEngineByteSpan { + sys::ApplicationEngineByteSpan { + data: payload.as_ptr(), + size: payload.len() as u64, + } +} + +/// # Safety +/// `engine` is a live handle. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn application_engine_last_executed_safe_block( + engine: *const ApplicationEngine, +) -> u64 { + unsafe { engine_ref(engine) } + .app + .progress() + .last_executed_safe_block() +} + +/// # Safety +/// `engine` is a live handle. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn application_engine_executed_input_count( + engine: *const ApplicationEngine, +) -> u64 { + unsafe { engine_ref(engine) } + .app + .progress() + .executed_input_count() + .get() +} + +/// # Safety +/// `engine` is a live handle and `prefix` is a NUL terminated path. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn application_engine_create_dump( + engine: *mut ApplicationEngine, + prefix: *const c_char, +) -> sys::ApplicationEngineStatus { + clear_error(); + let engine = unsafe { engine_mut(engine) }; + let prefix = unsafe { path_from(prefix) }; + match engine.app.create_dump(&prefix) { + Ok(()) => sys::APPLICATION_ENGINE_STATUS_OK, + Err(err) => report(&err, "create_dump"), + } +} + +/// # Safety +/// `prefix` is a NUL terminated path. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn application_engine_delete_dump( + prefix: *const c_char, +) -> sys::ApplicationEngineStatus { + clear_error(); + let prefix = unsafe { path_from(prefix) }; + match WalletApp::delete_dump(&prefix) { + Ok(()) => sys::APPLICATION_ENGINE_STATUS_OK, + Err(err) => report(&err, "delete_dump"), + } +} + +/// # Safety +/// `prefix` is a NUL terminated path. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn application_engine_state_file_in_dump( + prefix: *const c_char, +) -> *const c_char { + clear_error(); + let prefix = unsafe { path_from(prefix) }; + // Pure over the prefix: it touches no filesystem and needs no engine. Where the state file + // sits follows from the shape this app gives a dump, a directory with `state` inside it. + let state_file = WalletApp::state_file_in_dump(&prefix); + match CString::new(state_file.as_os_str().as_encoded_bytes()) { + Ok(path) => STATE_FILE.with(|slot| { + *slot.borrow_mut() = path; + slot.borrow().as_ptr() + }), + Err(_) => { + set_error("the dump prefix contains an interior NUL"); + std::ptr::null() + } + } +} + +/* -- genesis, the one entry point that is not part of the seam -- */ + +/// Write a genesis state at `prefix`, an empty wallet with the given deployment configuration. +/// +/// The seam has no create path, so every application ships a genesis tool. This is the wallet's. +pub fn write_genesis(prefix: &Path, config: WalletConfig) -> Result<(), AppError> { + WalletApp::new(config).create_dump(prefix) +} diff --git a/examples/c-wallet-engine/tests/conformance.rs b/examples/c-wallet-engine/tests/conformance.rs new file mode 100644 index 00000000..89d9ccd3 --- /dev/null +++ b/examples/c-wallet-engine/tests/conformance.rs @@ -0,0 +1,237 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +use alloy_primitives::{Address, U256}; +use app_core::application::{Method, Transfer, WalletApp, WalletConfig, Withdrawal}; +use c_app_engine::{Application, EngineApp}; +use c_wallet_engine as _; +use sequencer_core::application::{ + AppError, AppOutput, CanonicalState, ExecutionOutcome, execute_direct_input, + execute_valid_user_op, validate_and_execute_user_op, +}; +use sequencer_core::l2_tx::{DirectInput, ValidUserOp}; +use sequencer_core::user_op::UserOp; +use ssz::Encode; +use std::path::Path; + +fn fixture() -> (tempfile::TempDir, EngineApp, WalletApp, WalletConfig) { + let dir = tempfile::tempdir().unwrap(); + let config = WalletConfig::default(); + let genesis = dir.path().join("genesis"); + c_wallet_engine::write_genesis(&genesis, config).unwrap(); + let bridge = EngineApp::from_dump(&genesis).unwrap(); + let native = WalletApp::from_dump(&genesis).unwrap(); + (dir, bridge, native, config) +} + +fn state_bytes(app: &mut EngineApp, path: &Path) -> Vec { + app.create_dump(path).unwrap(); + std::fs::read(EngineApp::state_file_in_dump(path)).unwrap() +} + +fn deposit(config: WalletConfig, recipient: Address, amount: u64, block: u64) -> DirectInput { + let mut payload = Vec::new(); + payload.extend_from_slice(config.supported_erc20_token.as_slice()); + payload.extend_from_slice(recipient.as_slice()); + payload.extend_from_slice(&U256::from(amount).to_be_bytes::<32>()); + DirectInput { + sender: config.erc20_portal_address, + block_number: block, + payload, + } +} + +#[test] +fn abi_mixed_history_matches_native_outputs_progress_and_dump() { + let (dir, mut bridge, mut native, config) = fixture(); + let sender = Address::repeat_byte(0x11); + let recipient = Address::repeat_byte(0x22); + for direct in [ + deposit(config, sender, 1000, 7), + DirectInput { + sender: config.erc20_portal_address, + block_number: 4, + payload: vec![0xff], + }, + DirectInput { + sender, + block_number: 9, + payload: vec![], + }, + ] { + assert_eq!( + execute_direct_input(&mut bridge, &direct).unwrap(), + execute_direct_input(&mut native, &direct).unwrap() + ); + assert_eq!(bridge.progress(), native.progress()); + } + assert_eq!(bridge.progress().executed_input_count().get(), 3); + assert_eq!(bridge.progress().last_executed_safe_block(), 9); + + for (nonce, data, block) in [ + ( + 0, + Method::Transfer(Transfer { + amount: U256::from(100), + to: recipient, + }) + .as_ssz_bytes(), + 12, + ), + ( + 1, + Method::Withdrawal(Withdrawal { + amount: U256::from(50), + }) + .as_ssz_bytes(), + 10, + ), + (2, vec![0xff], 13), + ] { + let op = UserOp { + nonce, + max_fee: 10, + data: data.into(), + }; + let actual = validate_and_execute_user_op(&mut bridge, sender, &op, 10, block).unwrap(); + let expected = validate_and_execute_user_op(&mut native, sender, &op, 10, block).unwrap(); + assert_eq!(actual, expected); + let ExecutionOutcome::Included(receipt) = actual else { + panic!("funded operation was rejected") + }; + match nonce { + 0 => assert!(matches!(receipt.outputs.as_slice(), [AppOutput::Notice(_)])), + 1 => assert!( + matches!(receipt.outputs.as_slice(), [AppOutput::Voucher { value, .. }] if *value == U256::ZERO) + ), + 2 => assert!( + receipt.outputs.is_empty(), + "malformed method is an executed no-op" + ), + _ => unreachable!(), + } + assert_eq!(bridge.progress(), native.progress()); + } + assert_eq!(bridge.progress().executed_input_count().get(), 6); + assert_eq!(bridge.progress().last_executed_safe_block(), 13); + let checkpoint = dir.path().join("checkpoint"); + assert_eq!( + state_bytes(&mut bridge, &checkpoint), + native.canonical_snapshot_bytes().unwrap() + ); + let restored = EngineApp::from_dump(&checkpoint).unwrap(); + assert_eq!(restored.progress(), bridge.progress()); +} + +#[test] +fn abi_protocol_rejections_leave_state_and_progress_unchanged() { + let (dir, mut bridge, mut native, config) = fixture(); + let sender = Address::repeat_byte(0x11); + let direct = deposit(config, sender, 20, 7); + execute_direct_input(&mut bridge, &direct).unwrap(); + execute_direct_input(&mut native, &direct).unwrap(); + for (who, nonce, max_fee, current_fee) in [ + (sender, 9, 10, 10), + (sender, 0, 0, 10), + (Address::repeat_byte(0x55), 0, 10_000, 10_000), + ] { + let op = UserOp { + nonce, + max_fee, + data: vec![0xff].into(), + }; + let before = bridge.progress(); + let actual = validate_and_execute_user_op(&mut bridge, who, &op, current_fee, 99).unwrap(); + assert_eq!( + actual, + validate_and_execute_user_op(&mut native, who, &op, current_fee, 99).unwrap() + ); + assert!(matches!(actual, ExecutionOutcome::Invalid(_))); + assert_eq!(bridge.progress(), before); + } + assert_eq!( + state_bytes(&mut bridge, &dir.path().join("rejected")), + native.canonical_snapshot_bytes().unwrap() + ); +} + +#[test] +fn abi_restored_instances_and_checkpoints_are_independent() { + let (dir, mut first, _, config) = fixture(); + let source = dir.path().join("genesis"); + let mut second = EngineApp::from_dump(&source).unwrap(); + let original = std::fs::read(EngineApp::state_file_in_dump(&source)).unwrap(); + let input = deposit(config, Address::repeat_byte(0x11), 10, 7); + execute_direct_input(&mut first, &input).unwrap(); + assert_eq!(second.progress().executed_input_count().get(), 0); + assert_eq!( + std::fs::read(EngineApp::state_file_in_dump(&source)).unwrap(), + original + ); + let frozen = dir.path().join("frozen"); + let frozen_bytes = state_bytes(&mut first, &frozen); + execute_direct_input(&mut first, &input).unwrap(); + assert_eq!( + std::fs::read(EngineApp::state_file_in_dump(&frozen)).unwrap(), + frozen_bytes + ); + EngineApp::delete_dump(&source).unwrap(); + execute_direct_input(&mut second, &input).unwrap(); + assert_eq!( + state_bytes(&mut second, &dir.path().join("second")), + frozen_bytes + ); + drop(first); + let loaded = EngineApp::from_dump(&frozen).unwrap(); + assert_eq!(loaded.progress().executed_input_count().get(), 1); +} + +#[test] +fn abi_dump_errors_preserve_missing_and_corrupt_classification() { + let dir = tempfile::tempdir().unwrap(); + assert!( + matches!(EngineApp::from_dump(&dir.path().join("missing")), Err(AppError::Io(error)) if error.kind() == std::io::ErrorKind::NotFound) + ); + let corrupt = dir.path().join("corrupt"); + std::fs::create_dir(&corrupt).unwrap(); + std::fs::write(EngineApp::state_file_in_dump(&corrupt), [0xff]).unwrap(); + assert!( + matches!(EngineApp::from_dump(&corrupt), Err(AppError::Io(error)) if error.kind() == std::io::ErrorKind::InvalidData) + ); +} + +#[test] +fn abi_malformed_dump_paths_remain_terminal_like_native_reads() { + let dir = tempfile::tempdir().unwrap(); + let file_prefix = dir.path().join("file-prefix"); + std::fs::write(&file_prefix, []).unwrap(); + let directory_state = dir.path().join("directory-state"); + std::fs::create_dir(&directory_state).unwrap(); + std::fs::create_dir(WalletApp::state_file_in_dump(&directory_state)).unwrap(); + for prefix in [file_prefix, directory_state] { + assert!( + matches!(WalletApp::from_dump(&prefix), Err(AppError::Io(error)) + if matches!(error.kind(), std::io::ErrorKind::NotADirectory | std::io::ErrorKind::IsADirectory)) + ); + assert!( + matches!(EngineApp::from_dump(&prefix), Err(AppError::Io(error)) + if error.kind() == std::io::ErrorKind::InvalidData) + ); + } +} + +#[test] +fn abi_execution_failure_returns_app_error_without_resuming_the_instance() { + let (_dir, mut bridge, _, _) = fixture(); + let op = ValidUserOp { + sender: Address::repeat_byte(0x11), + fee: 10, + data: vec![], + }; + // Calling the already-validated boundary on an unfunded op is an application invariant fault. + assert!(matches!( + execute_valid_user_op(&mut bridge, &op, 7), + Err(AppError::Internal { .. }) + )); + drop(bridge); +} diff --git a/examples/c-wallet-sequencer/Cargo.toml b/examples/c-wallet-sequencer/Cargo.toml new file mode 100644 index 00000000..06080aa3 --- /dev/null +++ b/examples/c-wallet-sequencer/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "c-wallet-sequencer" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "Sequencer binary for the placeholder wallet app reached over the C API" +homepage.workspace = true +repository.workspace = true +readme = "../../README.md" +authors.workspace = true + +[dependencies] +c-app-sequencer = { path = "../c-app-sequencer" } +c-wallet-engine = { path = "../c-wallet-engine" } +tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } diff --git a/examples/c-wallet-sequencer/src/main.rs b/examples/c-wallet-sequencer/src/main.rs new file mode 100644 index 00000000..a418acc4 --- /dev/null +++ b/examples/c-wallet-sequencer/src/main.rs @@ -0,0 +1,16 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +//! The same wallet as `wallet-sequencer`, reached over the C seam instead of directly. This is +//! the model for what a C application author builds: depend on an engine, call the host. + +use std::process::ExitCode; + +// Load-bearing: nothing here calls the engine, but without the import the crate stays off the +// link line and the seam's symbols go unresolved. +use c_wallet_engine as _; + +#[tokio::main] +async fn main() -> ExitCode { + c_app_sequencer::run().await +} From 9b56dd79132d6a32eae2620ca7ca7e8a7c469406 Mon Sep 17 00:00:00 2001 From: gcdepaula Date: Wed, 9 Sep 2026 16:00:02 -0300 Subject: [PATCH 02/29] feat(harness)!: support lazy fallible genesis construction Let file-backed applications load genesis only when setup needs its initial snapshot. Completed setup remains a no-op after the original genesis file is gone. Expose run_command for custom parsers so the C host shares command-task panic projection and the existing bootstrap error policy. Remove the external engine's arbitrary payload ceiling and align checkpoint deletion documentation with database-reference removal before garbage collection. BREAKING CHANGE: genesis factories passed to run_main, run_command, dispatch, and setup return Result. Wrap infallible constructors in Ok(...). --- examples/c-app-engine/README.md | 13 +++++- examples/c-app-engine/build.rs | 9 ---- .../c-app-engine/include/application-engine.h | 4 +- examples/c-app-sequencer/src/lib.rs | 36 ++++++---------- .../src/bin/wallet-sequencer-devnet.rs | 2 +- examples/wallet-sequencer/src/main.rs | 2 +- sequencer/src/commands/setup/mod.rs | 14 +++---- sequencer/src/harness.rs | 41 ++++++++++++------- sequencer/src/lib.rs | 2 +- 9 files changed, 61 insertions(+), 62 deletions(-) diff --git a/examples/c-app-engine/README.md b/examples/c-app-engine/README.md index 89b72296..39825ad2 100644 --- a/examples/c-app-engine/README.md +++ b/examples/c-app-engine/README.md @@ -39,8 +39,17 @@ cargo test -p c-wallet-engine --test conformance ``` The ordinary setup/run environment configuration is still required. The genesis -path is required only for plain `setup`; warm startup, `flush-mempool`, and -`setup --recovery` use the sequencer's durable checkpoints. +path is required only when plain `setup` needs its first snapshot; completed +setup, warm startup, `flush-mempool`, and `setup --recovery` use the sequencer's +durable checkpoints. + +The host adds `--state-file` through its own parser and passes the parsed command +to `sequencer::run_command`. This shares `run_main`'s command lifecycle and exit +policy. Both take a lazy `FnOnce() -> Result` genesis factory; +infallible Rust constructors therefore use +`run_main(|| Ok(WalletApp::new(WalletConfig::default())))`. A missing genesis file +returns an ordinary application-bootstrap I/O error, while a caught factory +panic follows the shared terminal-error policy. ## External engine diff --git a/examples/c-app-engine/build.rs b/examples/c-app-engine/build.rs index af7f13ee..b7b9e677 100644 --- a/examples/c-app-engine/build.rs +++ b/examples/c-app-engine/build.rs @@ -17,9 +17,6 @@ use std::path::{Path, PathBuf}; /// rather than reaching a host. An application outside this workspace always declares its own. const REFERENCE_ENGINE_METHOD_PAYLOAD_LIMIT: u32 = 1 + 32 + 20; -/// A ceiling on what an application may declare, since the bound gates ingress. -const MAX_METHOD_PAYLOAD_LIMIT: u32 = 1 << 20; - /// Generate `sys`'s contents from the engine header, the one authoritative declaration of what /// the archive exports, so a change on the engine side is either picked up here or fails this /// build. @@ -123,12 +120,6 @@ fn external_engine(engine_lib: &str) -> (PathBuf, u32) { let limit = declared.trim().parse::().unwrap_or_else(|err| { panic!("APPLICATION_ENGINE_METHOD_PAYLOAD_LIMIT is not a number: {err}") }); - // The bound gates ingress and sizes batches, so a fat-fingered value is worth refusing here - // rather than discovering as a memory bill - assert!( - limit > 0 && limit <= MAX_METHOD_PAYLOAD_LIMIT, - "APPLICATION_ENGINE_METHOD_PAYLOAD_LIMIT is {limit}, expected 1..={MAX_METHOD_PAYLOAD_LIMIT}" - ); (header, limit) } diff --git a/examples/c-app-engine/include/application-engine.h b/examples/c-app-engine/include/application-engine.h index 6fea11ad..51c97669 100644 --- a/examples/c-app-engine/include/application-engine.h +++ b/examples/c-app-engine/include/application-engine.h @@ -376,8 +376,8 @@ APPLICATION_ENGINE_API ApplicationEngineStatus application_engine_create_dump(Ap /// @returns OK, NOT_FOUND when the dump is absent, IO_ERROR for other filesystem failures, /// or INTERNAL_ERROR. /// @details An engine still holding this dump open keeps running, its mapping outlives the name. -/// Synchronizing the directory entry before returning is what would let a caller drop its record -/// of the path on OK, so an engine that skips it can leave an orphan behind a crash. +/// The sequencer removes the database reference before deleting the artifact. Deletion must not +/// affect other dumps or independently loaded engines. APPLICATION_ENGINE_API ApplicationEngineStatus application_engine_delete_dump( const char *prefix) APPLICATION_ENGINE_NOEXCEPT; diff --git a/examples/c-app-sequencer/src/lib.rs b/examples/c-app-sequencer/src/lib.rs index 32c69f45..590fcea3 100644 --- a/examples/c-app-sequencer/src/lib.rs +++ b/examples/c-app-sequencer/src/lib.rs @@ -24,8 +24,8 @@ use tracing_subscriber::EnvFilter; All options can also be set via environment variables (shown in brackets)." )] struct Cli { - /// Engine genesis state, read-only and load-bearing for `setup` alone, `run` opens dumps. - /// Must already hold a deployment written by the application's genesis tool + /// Genesis dump used only when plain setup needs its initial snapshot. + /// Created by the application's genesis tool #[arg(long, env = "CARTESI_SEQUENCER_STATE_FILE", value_name = "PATH")] state_file: Option, #[command(subcommand)] @@ -47,26 +47,14 @@ pub async fn run() -> ExitCode { ) .init(); - // Only `setup` starts from this file, `run` and `flush-mempool` work from the dumps the - // sequencer took, so demanding it of them would keep a warm deployment from restarting once - // the genesis state is gone. Opened here rather than inside the closure so a state the engine - // cannot read names what to do about it instead of raising the closure's panic. - let mut app = None; - if matches!(&command, sequencer::Command::Setup(config) if !config.recovery) { - let Some(state_file) = state_file else { - tracing::error!("plain setup requires --state-file or CARTESI_SEQUENCER_STATE_FILE"); - return ExitCode::FAILURE; - }; - match EngineApp::from_dump(&state_file) { - Ok(engine) => app = Some(engine), - Err(err) => { - tracing::error!(state = %state_file.display(), - "cannot open the engine state, write one with the application's genesis tool first: {err:?}"); - return ExitCode::FAILURE; - } - } - } - - // Only plain setup invokes the genesis constructor. - sequencer::dispatch(command, move || app.expect("engine opened for setup")).await + sequencer::run_command(command, move || { + let state_file = state_file.ok_or_else(|| { + std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "plain setup requires --state-file or CARTESI_SEQUENCER_STATE_FILE", + ) + })?; + EngineApp::from_dump(&state_file) + }) + .await } diff --git a/examples/wallet-sequencer/src/bin/wallet-sequencer-devnet.rs b/examples/wallet-sequencer/src/bin/wallet-sequencer-devnet.rs index 0126ddc3..9921e177 100644 --- a/examples/wallet-sequencer/src/bin/wallet-sequencer-devnet.rs +++ b/examples/wallet-sequencer/src/bin/wallet-sequencer-devnet.rs @@ -20,5 +20,5 @@ async fn main() -> std::process::ExitCode { // `setup` is the only subcommand that constructs a genesis app; the // closure runs only on that path. - sequencer::run_main(|| WalletApp::new(WalletConfig::devnet())).await + sequencer::run_main(|| Ok(WalletApp::new(WalletConfig::devnet()))).await } diff --git a/examples/wallet-sequencer/src/main.rs b/examples/wallet-sequencer/src/main.rs index 40c53211..491fa25d 100644 --- a/examples/wallet-sequencer/src/main.rs +++ b/examples/wallet-sequencer/src/main.rs @@ -20,5 +20,5 @@ async fn main() -> std::process::ExitCode { // `setup` is the only subcommand that constructs a genesis app; the // closure runs only on that path. - sequencer::run_main(|| WalletApp::new(WalletConfig::default())).await + sequencer::run_main(|| Ok(WalletApp::new(WalletConfig::default()))).await } diff --git a/sequencer/src/commands/setup/mod.rs b/sequencer/src/commands/setup/mod.rs index 19b6d152..20ed23a5 100644 --- a/sequencer/src/commands/setup/mod.rs +++ b/sequencer/src/commands/setup/mod.rs @@ -25,7 +25,7 @@ //! key) and does no L1 writes. use alloy_primitives::Address; -use sequencer_core::application::Application; +use sequencer_core::application::{AppError, Application}; use sequencer_core::scheduler::{FoldInput, SchedulerConfig, fold_replay}; pub(crate) mod fill; @@ -43,7 +43,7 @@ use crate::storage::{self, DeploymentIdentity, FeeOracleIdentity}; pub async fn setup(config: SetupConfig, genesis_app: F) -> Result<(), CommandError> where A: Application + 'static, - F: FnOnce() -> A, + F: FnOnce() -> Result, { // Cross-field config validation (recovery vs the recovery-only args). A // misconfig is operator error — terminal, before any filesystem touch. @@ -83,7 +83,7 @@ async fn setup_admitted( ) -> Result<(), CommandError> where A: Application + 'static, - F: FnOnce() -> A, + F: FnOnce() -> Result, { let db_path = config.db_path(); let timing = config.timing.protocol_timing()?; @@ -332,11 +332,9 @@ where // ── Genesis snapshot ───────────────────────────────────── // Construct only after the admission facts and every - // detect-and-refuse gate. A panic leaves setup incomplete (the - // completion fact is never written, so the retry starts fresh), - // while completed no-ops and recovery never construct genesis - // state at all. - let genesis_app = genesis_app(); + // detect-and-refuse gate. Factory errors follow normal setup settlement; + // completed no-ops and recovery never construct genesis state. + let genesis_app = genesis_app()?; fill::register_genesis_finalized_snapshot::(genesis_app, &mut storage, &dumps_dir)?; } diff --git a/sequencer/src/harness.rs b/sequencer/src/harness.rs index 8f26d115..5df0c956 100644 --- a/sequencer/src/harness.rs +++ b/sequencer/src/harness.rs @@ -13,18 +13,17 @@ //! #[tokio::main] //! async fn main() -> std::process::ExitCode { //! init_tracing(); -//! sequencer::harness::run_main(|| WalletApp::new(WalletConfig::default())).await +//! sequencer::harness::run_main(|| Ok(WalletApp::new(WalletConfig::default()))).await //! } //! ``` //! -//! Genesis construction stays off the `Application` trait (it varies per impl) -//! and is supplied by this closure. When a future app needs setup-time CLI -//! args of its own (e.g. a machine-image path), the extension point is a -//! `Cli` generic on the parser — deferred until an app needs it, so -//! the harness imposes no `clap` bound on the (possibly FFI) app type. +//! Genesis construction stays off the `Application` trait (it varies per impl). +//! Apps with their own CLI options can parse [`Command`] and call [`run_command`] +//! with a lazy, fallible genesis factory. Both entry points share the command +//! lifecycle and exit policy without imposing a `clap` bound on the app type. use clap::{Parser, Subcommand}; -use sequencer_core::application::Application; +use sequencer_core::application::{AppError, Application}; use crate::commands::config::{FlushConfig, RunConfig, SetupConfig}; @@ -64,10 +63,20 @@ pub enum Command { pub async fn run_main(genesis_app: F) -> std::process::ExitCode where A: Application + 'static, - F: FnOnce() -> A + Send + 'static, + F: FnOnce() -> Result + Send + 'static, { let cli = Cli::parse(); - project_dispatch_join(tokio::spawn(dispatch(cli.command, genesis_app)).await) + run_command(cli.command, genesis_app).await +} + +/// Run a parsed command with the same panic and exit-code handling as [`run_main`]. +/// The fallible genesis factory runs only when plain setup needs its initial snapshot. +pub async fn run_command(command: Command, genesis_app: F) -> std::process::ExitCode +where + A: Application + 'static, + F: FnOnce() -> Result + Send + 'static, +{ + project_dispatch_join(tokio::spawn(dispatch(command, genesis_app)).await) } fn project_dispatch_join( @@ -100,11 +109,12 @@ fn project_dispatch_join( /// Terminal runtime faults abort the process directly; see [`crate::run`]. /// /// `genesis_app` is called at most once — only when plain `setup` needs to -/// register the genesis snapshot. +/// register the genesis snapshot. File-backed applications return load errors through +/// `AppError`; ordinary I/O failures are not invariant panics. pub async fn dispatch(command: Command, genesis_app: F) -> std::process::ExitCode where A: Application + 'static, - F: FnOnce() -> A, + F: FnOnce() -> Result, { let result = match command { Command::Setup(config) => crate::commands::setup::setup(*config, genesis_app).await, @@ -171,9 +181,12 @@ mod tests { let constructions = Arc::new(AtomicUsize::new(0)); let observed = Arc::clone(&constructions); - let exit = dispatch::(cli.command, move || { + let exit = run_command::(cli.command, move || { observed.fetch_add(1, Ordering::SeqCst); - SweepTestApp + Err(AppError::Io(std::io::Error::new( + std::io::ErrorKind::NotFound, + "genesis file no longer exists", + ))) }) .await; @@ -209,7 +222,7 @@ mod tests { ]) .expect("parse run"); - let exit = dispatch::(cli.command, || SweepTestApp).await; + let exit = dispatch::(cli.command, || Ok(SweepTestApp)).await; assert_eq!(exit, std::process::ExitCode::from(30)); } diff --git a/sequencer/src/lib.rs b/sequencer/src/lib.rs index 420746bc..5c93c895 100644 --- a/sequencer/src/lib.rs +++ b/sequencer/src/lib.rs @@ -37,5 +37,5 @@ mod integration_tests; pub use commands::config::{FlushConfig, RunConfig, SetupConfig}; pub use commands::error::CommandError; pub use commands::run::run; -pub use harness::{Cli, Command, dispatch, run_main}; +pub use harness::{Cli, Command, dispatch, run_command, run_main}; pub use http::{ApiConfig, ApiError, WS_CATCHUP_WINDOW_EXCEEDED_REASON}; From 84dcdd77fe706e876e502276e277a00152a90391 Mon Sep 17 00:00:00 2001 From: gcdepaula Date: Fri, 11 Sep 2026 10:34:47 -0300 Subject: [PATCH 03/29] test(ffi): cover output ownership and external archive builds Exercise mixed outputs through a test ABI that reuses its payload buffer, including a full-width nonzero voucher value. Restore the external archive CI smoke check and libclang dependency, and clarify bridge discovery and cross-target determinism responsibilities. Validation: workspace check, strict Clippy, formatting, 718 host tests, and external archive build plus CLI smoke check passed. --- .github/workflows/ci.yml | 10 ++ README.md | 1 + docs/protocol/application-contract.md | 2 + examples/c-app-engine/README.md | 2 + .../c-app-engine/include/application-engine.h | 8 +- examples/c-app-engine/src/lib.rs | 3 + examples/c-app-engine/src/tests.rs | 103 ++++++++++++++++++ 7 files changed, 125 insertions(+), 4 deletions(-) create mode 100644 examples/c-app-engine/src/tests.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b1e0f12f..36212552 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,6 +24,7 @@ jobs: sudo apt-get update sudo apt-get install -y \ faketime \ + libclang-dev \ libfaketime \ lua5.4 \ liblua5.4-dev \ @@ -62,6 +63,15 @@ jobs: timeout-minutes: 15 run: cargo test --workspace --all-targets --all-features --locked + - name: C application archive path + run: | + cargo build --locked -p c-wallet-engine + APPLICATION_ENGINE_LIB="$PWD/target/debug/libc_wallet_engine.a" \ + APPLICATION_ENGINE_HEADER="$PWD/examples/c-app-engine/include/application-engine.h" \ + APPLICATION_ENGINE_METHOD_PAYLOAD_LIMIT=53 \ + cargo build --locked -p c-app-sequencer + ./target/debug/c-app-sequencer --help > /dev/null + canonical-guest: runs-on: ubuntu-latest needs: rust diff --git a/README.md b/README.md index a571cc1d..810d8598 100644 --- a/README.md +++ b/README.md @@ -260,6 +260,7 @@ released even on client disconnect. - `sequencer/src/storage/`: schema, migrations, SQLite persistence (split per writer role), and replay reads - `sequencer-core/src/`: shared domain types and interfaces (`Application`, `SignedUserOp`, `SequencedL2Tx`, feed message types) - `examples/app-core/src/`: wallet prototype implementing `Application` +- [`examples/c-app-engine/`](examples/c-app-engine/README.md): C ABI bridge, reference wallet engine, and external static-archive integration guide - `tests/benchmarks/`: benchmark harnesses and benchmark spec Related docs: diff --git a/docs/protocol/application-contract.md b/docs/protocol/application-contract.md index bbdb8131..2f5b9253 100644 --- a/docs/protocol/application-contract.md +++ b/docs/protocol/application-contract.md @@ -11,6 +11,8 @@ of a native engine, its FFI, or its canonical counterpart This document **owns** the contract. [`AGENTS.md`](../../AGENTS.md) is the map. The [wallet](../../examples/app-core/) is the reference implementation. A production application may execute natively or wrap a Cartesi Machine. +The [C application bridge](../../examples/c-app-engine/README.md) adapts a native +engine through a C ABI and includes a reference wallet integration. ## The execution methods diff --git a/examples/c-app-engine/README.md b/examples/c-app-engine/README.md index 39825ad2..705299c9 100644 --- a/examples/c-app-engine/README.md +++ b/examples/c-app-engine/README.md @@ -71,3 +71,5 @@ archive configured, the generic binary reports that no engine was linked, and The conformance suite compares native and ABI execution over mixed inputs, notices and vouchers, rejection/no-op progress, dump round trips, independent instances, and fatal/error classification. +`cargo test -p c-app-engine --lib` also checks mixed-output ordering, copying +reused engine buffers, and full-width voucher values with a small ABI fixture. diff --git a/examples/c-app-engine/include/application-engine.h b/examples/c-app-engine/include/application-engine.h index 51c97669..6adeb81f 100644 --- a/examples/c-app-engine/include/application-engine.h +++ b/examples/c-app-engine/include/application-engine.h @@ -12,10 +12,10 @@ /// agnostic and an engine is swappable behind this header. Plain C so any host can consume it. /// /// An application implements these declarations into a static archive, which the `c-app-engine` -/// shim links at build time to become the sequencer's Application, and which the application's -/// own canonical binary links natively from the same objects. One compiled engine on both sides -/// is what makes off-chain and on-chain execution deterministic. An application written in C++, -/// or in any language with a C ABI, implements it the same way. +/// shim links at build time to become the sequencer's Application. The same engine implementation +/// can be compiled for the native sequencer and the application's canonical target; deterministic, +/// equivalent behavior across those builds remains the application's responsibility. An engine +/// written in C++, or in any language with a C ABI, implements it the same way. /// /// Nothing application specific crosses. An engine is handed a dump already holding a configured /// deployment, so a host never learns what configures the application it runs, and the path is diff --git a/examples/c-app-engine/src/lib.rs b/examples/c-app-engine/src/lib.rs index b2edc38d..24a4d223 100644 --- a/examples/c-app-engine/src/lib.rs +++ b/examples/c-app-engine/src/lib.rs @@ -5,6 +5,9 @@ pub mod sys; +#[cfg(test)] +mod tests; + use std::ffi::{CStr, CString, OsStr}; use std::os::unix::ffi::OsStrExt; use std::path::{Path, PathBuf}; diff --git a/examples/c-app-engine/src/tests.rs b/examples/c-app-engine/src/tests.rs new file mode 100644 index 00000000..c656da7c --- /dev/null +++ b/examples/c-app-engine/src/tests.rs @@ -0,0 +1,103 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +use super::*; + +// Only the drain seam is needed here; wallet integration tests cover execution and dumps. +// Every non-empty output borrows the same allocation, overwritten by the next drain. +#[derive(Default)] +struct OutputEngine { + drained: usize, + payload: [u8; 4], +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn application_engine_drain_output( + engine: *mut sys::ApplicationEngine, + out_output: *mut sys::ApplicationEngineOutput, +) -> i32 { + // SAFETY: the test transfers a boxed OutputEngine to EngineApp; it remains exclusively + // owned until destroy. EngineApp supplies a writable output record for every call. + let engine = unsafe { &mut *engine.cast::() }; + engine.payload = match engine.drained { + 0 => *b"one!", + 1 => [0, 0xff, 0x80, 0x42], + _ => *b"last", + }; + let (kind, values) = match engine.drained { + 0 | 3 => ( + sys::APPLICATION_ENGINE_OUTPUT_NOTICE, + sys::ApplicationEngineOutputValues { + notice: abi_span(&engine.payload), + }, + ), + 1 => ( + sys::APPLICATION_ENGINE_OUTPUT_VOUCHER, + sys::ApplicationEngineOutputValues { + voucher: sys::ApplicationEngineVoucher { + destination: sys::ApplicationEngineEthereumAddress { bytes: [0x23; 20] }, + value: sys::ApplicationEngineUint256 { + bytes: [ + 129, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, + 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, + ], + }, + payload: abi_span(&engine.payload[..3]), + }, + }, + ), + 2 => ( + sys::APPLICATION_ENGINE_OUTPUT_NOTICE, + sys::ApplicationEngineOutputValues { + notice: sys::ApplicationEngineByteSpan { + data: std::ptr::null(), + size: 0, + }, + }, + ), + _ => return sys::APPLICATION_ENGINE_STATUS_INTERNAL_ERROR, + }; + engine.drained += 1; + unsafe { *out_output = sys::ApplicationEngineOutput { kind, values } }; + sys::APPLICATION_ENGINE_STATUS_OK +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn application_engine_destroy(engine: *mut sys::ApplicationEngine) { + // SAFETY: EngineApp calls this once for the allocation transferred by the test. + drop(unsafe { Box::from_raw(engine.cast::()) }); +} + +#[unsafe(no_mangle)] +extern "C" fn application_engine_get_last_error_message() -> *const std::ffi::c_char { + c"output fixture exhausted".as_ptr() +} + +#[test] +fn drain_preserves_output_order_and_copies_reused_payloads() { + let mut app = EngineApp { + engine: Box::into_raw(Box::::default()).cast(), + }; + let outputs = app.drain_outputs(4).unwrap(); + drop(app); + + assert_eq!( + outputs, + vec![ + AppOutput::Notice(b"one!".to_vec()), + AppOutput::Voucher { + destination: Address::repeat_byte(0x23), + // Independent of the fixture's big-endian bytes: limbs are least-significant first. + value: U256::from_limbs([ + 0x191a1b1c1d1e1f20, + 0x1112131415161718, + 0x090a0b0c0d0e0f10, + 0x8102030405060708, + ]), + payload: vec![0, 0xff, 0x80], + }, + AppOutput::Notice(vec![]), + AppOutput::Notice(b"last".to_vec()), + ] + ); +} From d19dc97434cab218723e061df8729d600ad0d21e Mon Sep 17 00:00:00 2001 From: gcdepaula Date: Fri, 11 Sep 2026 21:25:14 -0300 Subject: [PATCH 04/29] fix(runtime): contain application lifecycle failures Catch application snapshot-path panics at both HTTP handlers and invoke the host terminal-abort policy. Classify deterministic application bootstrap I/O failures as terminal while keeping operational I/O restartable and genesis construction lazy. Validate both snapshot routes with subprocess SIGABRT tests and extend the exit-code table with terminal and retryable bootstrap cases. --- sequencer/src/commands/error.rs | 40 ++++++++++++++++++ sequencer/src/egress/api/snapshot.rs | 62 +++++++++++++++++++++++++++- 2 files changed, 100 insertions(+), 2 deletions(-) diff --git a/sequencer/src/commands/error.rs b/sequencer/src/commands/error.rs index 4194cebd..0a2034c0 100644 --- a/sequencer/src/commands/error.rs +++ b/sequencer/src/commands/error.rs @@ -156,6 +156,21 @@ impl CommandError { CommandError::AppBootstrap(AppError::Internal { .. }) => { CommandFailureVerdict::Terminal } + CommandError::AppBootstrap(AppError::Io(source)) + if matches!( + source.kind(), + std::io::ErrorKind::InvalidInput + | std::io::ErrorKind::NotFound + | std::io::ErrorKind::PermissionDenied + | std::io::ErrorKind::InvalidData + | std::io::ErrorKind::UnexpectedEof + | std::io::ErrorKind::IsADirectory + | std::io::ErrorKind::NotADirectory + ) => + { + // An absent option or unusable genesis dump needs operator repair. + CommandFailureVerdict::Terminal + } CommandError::ReferencedSnapshotArtifact { source, .. } if referenced_artifact_io_is_terminal(source) => { @@ -951,6 +966,23 @@ mod tests { ) }), ); + rows.extend( + [ + std::io::ErrorKind::InvalidInput, + std::io::ErrorKind::NotFound, + std::io::ErrorKind::PermissionDenied, + std::io::ErrorKind::InvalidData, + std::io::ErrorKind::UnexpectedEof, + std::io::ErrorKind::IsADirectory, + std::io::ErrorKind::NotADirectory, + ] + .map(|kind| { + ( + CommandError::AppBootstrap(AppError::Io(std::io::Error::from(kind))), + "a missing option or unusable application genesis dump", + ) + }), + ); rows.extend([ // A deterministic signer-construction misconfig (bad RPC URL or // private key) classifies terminal in every command, matching the @@ -1252,6 +1284,14 @@ mod tests { CommandError::AppBootstrap(AppError::Io(std::io::Error::other("disk unavailable"))), "an application I/O failure at bootstrap", ), + ( + CommandError::AppBootstrap(AppError::Io(std::io::ErrorKind::Interrupted.into())), + "an interrupted genesis read remains restartable", + ), + ( + CommandError::AppBootstrap(AppError::Io(std::io::ErrorKind::TimedOut.into())), + "a timed out genesis read remains restartable", + ), ] } diff --git a/sequencer/src/egress/api/snapshot.rs b/sequencer/src/egress/api/snapshot.rs index 58d9e930..b7dc4a8c 100644 --- a/sequencer/src/egress/api/snapshot.rs +++ b/sequencer/src/egress/api/snapshot.rs @@ -128,7 +128,7 @@ async fn finalized_state( return StatusCode::NOT_MODIFIED.into_response(); } - let path = (state.snapshot.state_file_in_dump)(&leased.prefix); + let path = state_file_path(&state.snapshot, &leased.prefix); let l2_tx_index = leased.l2_tx_index; let LeasedDump { guard, .. } = leased; @@ -162,7 +162,7 @@ async fn latest_snapshot(State(state): State>) -> Response Err(err) => return internal_error("acquire latest snapshot lease", err), }; - let path = (state.snapshot.state_file_in_dump)(&leased.prefix); + let path = state_file_path(&state.snapshot, &leased.prefix); let l2_tx_index = leased.l2_tx_index; let LeasedDump { guard, .. } = leased; @@ -184,6 +184,12 @@ async fn latest_snapshot(State(state): State>) -> Response } } +fn state_file_path(state: &SnapshotState, prefix: &Path) -> PathBuf { + // HTTP request panics are otherwise isolated from the worker supervisor. + std::panic::catch_unwind(|| (state.state_file_in_dump)(prefix)) + .unwrap_or_else(|_| abort_terminal("application snapshot path callback panicked")) +} + fn stream_body(file: File, guard: LeaseGuard) -> Body { Body::from_stream(ReaderStream::new(GuardedReader { file, @@ -273,6 +279,58 @@ mod tests { use super::*; use crate::storage::test_helpers::temp_db; + #[cfg(unix)] + async fn panicking_state_path_aborts(test_name: &str, finalized: bool) { + if !crate::runtime::shutdown::abort_test_child(test_name) { + return; + } + let db = temp_db("panicking-snapshot-path"); + let mut storage = Storage::open(&db.path).expect("open storage"); + storage + .insert_finalized_dump(Path::new("/tmp/panicking-snapshot-path"), 12, 34) + .expect("insert finalized snapshot"); + drop(storage); + let state = Arc::new(SnapshotApiState { + snapshot: SnapshotState { + db_path: db.path, + state_file_in_dump: |_| panic!("application returned no state path"), + }, + shutdown: RuntimeScope::default(), + release_scheduler: Arc::new(|release| release()), + }); + + // Exercise the request-task boundary that would swallow this panic. + let result = tokio::spawn(async move { + if finalized { + finalized_state(State(state), HeaderMap::new()).await + } else { + latest_snapshot(State(state)).await + } + }) + .await; + panic!("snapshot path callback panic did not abort the process: {result:?}"); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + #[cfg(unix)] + async fn finalized_state_path_panic_aborts_process() { + panicking_state_path_aborts( + "egress::api::snapshot::tests::finalized_state_path_panic_aborts_process", + true, + ) + .await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + #[cfg(unix)] + async fn latest_snapshot_path_panic_aborts_process() { + panicking_state_path_aborts( + "egress::api::snapshot::tests::latest_snapshot_path_panic_aborts_process", + false, + ) + .await; + } + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[cfg(unix)] async fn corrupt_finalized_snapshot_trips_terminal_storage_fault() { From f8f2ebad36c364883110396b2bfb9cead90ef3cb Mon Sep 17 00:00:00 2001 From: gcdepaula Date: Fri, 11 Sep 2026 21:25:14 -0300 Subject: [PATCH 05/29] fix(examples): handle genesis help without creating state Recognize --help and -h before treating the argument as a dump path. Pin successful help output and an untouched working directory in a CLI regression test, and remove the unused tracing dependency from the generic host. --- Cargo.lock | 1 - examples/c-app-sequencer/Cargo.toml | 1 - .../c-wallet-engine/src/bin/c-wallet-genesis.rs | 12 ++++++++---- examples/c-wallet-engine/tests/genesis_cli.rs | 17 +++++++++++++++++ 4 files changed, 25 insertions(+), 6 deletions(-) create mode 100644 examples/c-wallet-engine/tests/genesis_cli.rs diff --git a/Cargo.lock b/Cargo.lock index bdac5421..18a0ed38 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1391,7 +1391,6 @@ dependencies = [ "clap", "sequencer", "tokio", - "tracing", "tracing-subscriber", ] diff --git a/examples/c-app-sequencer/Cargo.toml b/examples/c-app-sequencer/Cargo.toml index 0b04b800..050d9356 100644 --- a/examples/c-app-sequencer/Cargo.toml +++ b/examples/c-app-sequencer/Cargo.toml @@ -18,5 +18,4 @@ c-app-engine = { path = "../c-app-engine" } sequencer = { path = "../../sequencer" } clap = { workspace = true, features = ["env"] } tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } -tracing = { workspace = true } tracing-subscriber = { workspace = true } diff --git a/examples/c-wallet-engine/src/bin/c-wallet-genesis.rs b/examples/c-wallet-engine/src/bin/c-wallet-genesis.rs index ddb63090..b8810c08 100644 --- a/examples/c-wallet-engine/src/bin/c-wallet-genesis.rs +++ b/examples/c-wallet-engine/src/bin/c-wallet-genesis.rs @@ -9,17 +9,21 @@ use std::process::ExitCode; use app_core::application::WalletConfig; +const USAGE: &str = "usage: c-wallet-genesis [devnet|sepolia]\n\n\ + Writes a genesis wallet state at , which must not already exist."; + fn main() -> ExitCode { let arguments: Vec = std::env::args().skip(1).collect(); + if matches!(arguments.as_slice(), [flag] if flag == "--help" || flag == "-h") { + println!("{USAGE}"); + return ExitCode::SUCCESS; + } let config = match arguments.as_slice() { [_, preset] if preset == "devnet" => WalletConfig::devnet(), [_, preset] if preset == "sepolia" => WalletConfig::sepolia(), [_] => WalletConfig::default(), _ => { - eprintln!( - "usage: c-wallet-genesis [devnet|sepolia]\n\n\ - Writes a genesis wallet state at , which must not already exist." - ); + eprintln!("{USAGE}"); return ExitCode::from(2); } }; diff --git a/examples/c-wallet-engine/tests/genesis_cli.rs b/examples/c-wallet-engine/tests/genesis_cli.rs new file mode 100644 index 00000000..817ad042 --- /dev/null +++ b/examples/c-wallet-engine/tests/genesis_cli.rs @@ -0,0 +1,17 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +#[test] +fn help_does_not_create_a_genesis_dump() { + let dir = tempfile::tempdir().unwrap(); + for flag in ["--help", "-h"] { + let output = std::process::Command::new(env!("CARGO_BIN_EXE_c-wallet-genesis")) + .arg(flag) + .current_dir(dir.path()) + .output() + .unwrap(); + assert!(output.status.success()); + assert!(String::from_utf8(output.stdout).unwrap().contains("usage:")); + assert_eq!(std::fs::read_dir(dir.path()).unwrap().count(), 0); + } +} From a4e300c6b8f58274c88d64cbbdcf3bec3d56654f Mon Sep 17 00:00:00 2001 From: gcdepaula Date: Fri, 11 Sep 2026 21:25:48 -0300 Subject: [PATCH 06/29] refactor(application)!: simplify the native engine contract Read the stable method payload bound from the linked engine, return count and clock together, and let the sequencer discard self-contained checkpoint directories with ordinary filesystem deletion. Remove the duplicated payload-limit build setting and cleanup-only Application type parameters. Update the reference ABI implementation, file/directory independence tests, build instructions, and protocol guide. Keep engine-dependent output, layout, conformance, and linking work documented as follow-ups. BREAKING CHANGE: Application implementations replace MAX_METHOD_PAYLOAD_BYTES with max_method_payload_bytes() and remove delete_dump. Native archives export application_engine_max_method_payload_bytes and application_engine_progress with ApplicationEngineProgress instead of scalar progress getters and a deletion hook. External builds require only the archive and header settings. Validation: 722 host tests passed, with one existing ignored doctest; workspace check, strict Clippy, formatting, C11/C++17 header checks, and external archive build plus CLI smoke check passed. --- .github/workflows/ci.yml | 1 - AGENTS.md | 8 +- CLAUDE.md | 6 +- README.md | 7 +- docs/protocol/application-contract.md | 22 ++- docs/protocol/c-application-binding.md | 49 +++++++ docs/review/register.md | 32 +++++ docs/snapshots/README.md | 2 +- docs/snapshots/format.md | 9 +- docs/snapshots/lifecycle.md | 7 +- examples/app-core/src/application/wallet.rs | 19 ++- examples/c-app-engine/README.md | 21 ++- examples/c-app-engine/build.rs | 45 ++----- .../c-app-engine/include/application-engine.h | 126 ++++++++---------- examples/c-app-engine/src/lib.rs | 28 ++-- examples/c-app-engine/src/tests.rs | 12 +- examples/c-app-sequencer/src/main.rs | 3 +- examples/c-wallet-engine/src/lib.rs | 57 +++----- examples/c-wallet-engine/tests/conformance.rs | 7 +- sequencer-core/src/application/mod.rs | 19 +-- sequencer-core/src/scheduler/fold.rs | 7 +- sequencer-core/src/scheduler/mod.rs | 8 +- sequencer/src/commands/run/startup_hygiene.rs | 25 ++-- sequencer/src/commands/run/workers.rs | 15 +-- sequencer/src/commands/setup/fill.rs | 8 +- sequencer/src/commands/test_support.rs | 11 +- sequencer/src/http.rs | 2 +- .../src/ingress/inclusion_lane/dump_info.rs | 46 +++---- sequencer/src/ingress/inclusion_lane/mod.rs | 5 +- .../src/ingress/inclusion_lane/snapshot.rs | 25 ++-- sequencer/src/ingress/inclusion_lane/tests.rs | 45 +++---- 31 files changed, 347 insertions(+), 330 deletions(-) create mode 100644 docs/protocol/c-application-binding.md diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 36212552..803a2a88 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -68,7 +68,6 @@ jobs: cargo build --locked -p c-wallet-engine APPLICATION_ENGINE_LIB="$PWD/target/debug/libc_wallet_engine.a" \ APPLICATION_ENGINE_HEADER="$PWD/examples/c-app-engine/include/application-engine.h" \ - APPLICATION_ENGINE_METHOD_PAYLOAD_LIMIT=53 \ cargo build --locked -p c-app-sequencer ./target/debug/c-app-sequencer --help > /dev/null diff --git a/AGENTS.md b/AGENTS.md index 78c6c5db..679cef38 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -147,6 +147,10 @@ Top-level layout follows the system's data flow. Each sequencer module correspon - `sequencer-core/` — shared domain types (`Application`, `SignedUserOp`, `SequencedL2Tx`, `Batch`, `Frame`). - `examples/app-core/` — placeholder wallet app implementing the `Application` trait. - `examples/wallet-sequencer/` — binary crate: wallet app + sequencer library. The model for what an app author builds (their `Application` impl ≙ `app-core`; their binary crate ≙ this). +- `examples/c-app-engine/` — C ABI adapter implementing `Application` for a native engine. +- `examples/c-app-sequencer/` — shared C-engine CLI host and external-archive binary. +- `examples/c-wallet-engine/` — reference wallet engine exporting the C ABI, plus its genesis tool and conformance tests. +- `examples/c-wallet-sequencer/` — binary composing the C-engine host with the reference wallet engine. - `examples/canonical-app/` — on-chain scheduler reference implementation. - `examples/canonical-test/` — e2e test harness for the canonical app. - `sdk/rust-client/` — Rust client library for the sequencer API. @@ -247,7 +251,7 @@ Logical state changes, including `ApplicationProgress`, flow through the `apply_ User ops are executed only through `sequencer_core::application::validate_and_execute_user_op`; already-validated user ops and directs use `execute_valid_user_op` / `execute_direct_input`. The shared boundary preflights the checked successor, then verifies the engine's progress after a successful hook and returns its pre-execution offset. Count zero implies clock zero. Validation purity and native mutation remain self-trusted. `AppError` is fatal and defines no canonical successor; callers discard the instance rather than resume it. The inclusion lane, canonical scheduler, catch-up, and recovery fold all use this boundary — part of the duality agreement. -`Application` requires `Send`, with neither `Clone` nor `Sync`. Dumps must be durable and immutable, and restored engines must remain independent after source deletion. The opaque app prefix may be a file or directory. Canonical inspection belongs to the separate `CanonicalState` trait; the native sequencer serves the comparison file in the checkpoint. +`Application` requires `Send`, with neither `Clone` nor `Sync`. Dumps must be durable and immutable, and restored engines must remain independent after source deletion. The opaque app prefix may be a file or directory; checkpoint disposal uses ordinary recursive filesystem deletion. Canonical inspection belongs to the separate `CanonicalState` trait; the native sequencer serves the comparison file in the checkpoint. The [C binding guide](docs/protocol/c-application-binding.md) maps the contract to native engines. ## Hot-Path Invariants @@ -455,7 +459,7 @@ Before finishing a change, ensure: - [`README.md`](README.md) — product framing, user-facing trust model, **API contract** (endpoint shapes, caps, close codes, health semantics). - [`CLAUDE.md`](CLAUDE.md) — shell setup, quick reference, pointer back here. -- [`docs/protocol/`](docs/protocol/) — the authoritative protocol contracts: [`scheduler-semantics.md`](docs/protocol/scheduler-semantics.md) (the canonical acceptance algorithm, I1) and [`application-contract.md`](docs/protocol/application-contract.md) (the `Application` FFI trait contract). +- [`docs/protocol/`](docs/protocol/) — the authoritative protocol contracts: [`scheduler-semantics.md`](docs/protocol/scheduler-semantics.md) (the canonical acceptance algorithm, I1), [`application-contract.md`](docs/protocol/application-contract.md) (the `Application` trait contract), and [`c-application-binding.md`](docs/protocol/c-application-binding.md) (the native C binding). - [`docs/invariants.md`](docs/invariants.md) — register of cross-module invariants (what's load-bearing across files) + the fail-loud check policy. - [`docs/review/register.md`](docs/review/register.md) — the review register: open findings, settled decisions, refuted proposals (do-not-re-propose), and the review history table; the dated ledgers beside it carry the evidence the table points at. - [`docs/plans/`](docs/plans/) — the architecture decision record ([`2026-08-authority-boundary-adr.md`](docs/plans/2026-08-authority-boundary-adr.md)), active coordination tracks, and in-flight design handoffs. diff --git a/CLAUDE.md b/CLAUDE.md index 56824121..f27ecb8d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -34,6 +34,10 @@ Rust edition 2024 / Axum API / SQLite (rusqlite, WAL) / EIP-712 signing / SSZ en - `sequencer-core/` — shared domain types consumed by both sequencer and scheduler. - `examples/app-core/` — placeholder wallet app implementing `Application`. - `examples/wallet-sequencer/` — binary crate: wallet app + sequencer library. +- `examples/c-app-engine/` — native engine adapter implementing `Application` through a C ABI. +- `examples/c-app-sequencer/` — shared C-engine CLI host and external-archive binary. +- `examples/c-wallet-engine/` — reference C ABI exports, genesis tool, and conformance tests. +- `examples/c-wallet-sequencer/` — binary composing the C-engine host with the reference wallet engine. - `examples/canonical-app/` — on-chain scheduler reference implementation. - `examples/canonical-test/` — e2e test harness for the canonical app. - `sdk/rust-client/` — Rust client library for the sequencer API. @@ -58,7 +62,7 @@ Rust edition 2024 / Axum API / SQLite (rusqlite, WAL) / EIP-712 signing / SSZ en ## Before You Start Real Work - **[`AGENTS.md`](AGENTS.md)** — mission, requirements, invariants, duality, recovery, conventions, rules. -- **[`docs/protocol/`](docs/protocol/)** — the authoritative protocol contracts: [`scheduler-semantics.md`](docs/protocol/scheduler-semantics.md) (canonical acceptance algorithm) and [`application-contract.md`](docs/protocol/application-contract.md) (the `Application` FFI trait). Read before touching the scheduler, the gold frontier, the fold, or an `Application` impl. +- **[`docs/protocol/`](docs/protocol/)** — the authoritative protocol contracts: [`scheduler-semantics.md`](docs/protocol/scheduler-semantics.md) (canonical acceptance algorithm), [`application-contract.md`](docs/protocol/application-contract.md) (the `Application` trait), and [`c-application-binding.md`](docs/protocol/c-application-binding.md) (the native C binding). Read before touching the scheduler, the gold frontier, the fold, or an `Application` impl. - **[`docs/invariants.md`](docs/invariants.md)** — cross-module invariants register + the fail-loud check policy. Check it before changing anything it lists as load-bearing. - **[`docs/review/register.md`](docs/review/register.md)** — the review register: open findings, settled decisions, refuted proposals (do-not-re-propose). Check it for open findings in code you're about to touch, and before proposing a mechanism or simplification. - **[`docs/plans/`](docs/plans/)** — the [authority-boundary ADR](docs/plans/2026-08-authority-boundary-adr.md), active coordination tracks, and in-flight design handoffs. Check before starting work that might belong to a track. diff --git a/README.md b/README.md index 810d8598..a0920cfe 100644 --- a/README.md +++ b/README.md @@ -260,10 +260,15 @@ released even on client disconnect. - `sequencer/src/storage/`: schema, migrations, SQLite persistence (split per writer role), and replay reads - `sequencer-core/src/`: shared domain types and interfaces (`Application`, `SignedUserOp`, `SequencedL2Tx`, feed message types) - `examples/app-core/src/`: wallet prototype implementing `Application` -- [`examples/c-app-engine/`](examples/c-app-engine/README.md): C ABI bridge, reference wallet engine, and external static-archive integration guide +- [`examples/c-app-engine/`](examples/c-app-engine/README.md): C ABI adapter and external static-archive integration guide +- `examples/c-app-sequencer/`: shared C-engine CLI host and external-archive binary +- `examples/c-wallet-engine/`: reference wallet C ABI exports, genesis tool, and conformance tests +- `examples/c-wallet-sequencer/`: binary composing the C-engine host with the reference wallet engine - `tests/benchmarks/`: benchmark harnesses and benchmark spec Related docs: + +- C application binding: [`docs/protocol/c-application-binding.md`](docs/protocol/c-application-binding.md) - App snapshots (format + lifecycle): `docs/snapshots/` - Watchdog — local dev: [`docs/watchdog/getting-started.md`](docs/watchdog/getting-started.md); Sepolia/mainnet: [`docs/watchdog/operator-deployment.md`](docs/watchdog/operator-deployment.md) diff --git a/docs/protocol/application-contract.md b/docs/protocol/application-contract.md index 2f5b9253..8d312ba6 100644 --- a/docs/protocol/application-contract.md +++ b/docs/protocol/application-contract.md @@ -11,8 +11,9 @@ of a native engine, its FFI, or its canonical counterpart This document **owns** the contract. [`AGENTS.md`](../../AGENTS.md) is the map. The [wallet](../../examples/app-core/) is the reference implementation. A production application may execute natively or wrap a Cartesi Machine. -The [C application bridge](../../examples/c-app-engine/README.md) adapts a native -engine through a C ABI and includes a reference wallet integration. +The [C application binding](c-application-binding.md) adapts this contract to +native engines; its [build guide](../../examples/c-app-engine/README.md) includes +a reference wallet integration. ## The execution methods @@ -37,9 +38,10 @@ Overflow fails before the hook runs. An error defines no successor: callers terminate the execution path and discard the instance, without attempting to roll back or inspect partially updated state. -`MAX_METHOD_PAYLOAD_BYTES` is both an ingress payload limit and a batch-sizing -input. HTTP rejects oversized method payloads; the lane uses the declared bound -plus signed-op metadata to compute batch capacity. It is not a canonical +`Application::max_method_payload_bytes()` returns the engine's stable method +payload bound without constructing an instance. Zero permits only empty method +payloads. HTTP rejects oversized method payloads; the lane uses the declared +bound plus signed-op metadata to compute batch capacity. It is not a canonical scheduler rejection rule. ## Cross-cutting contracts @@ -153,8 +155,11 @@ machine checkpoint may instead contain a separate app-state projection. The possibly `prefix` itself. Its bytes match the canonical application's deterministic comparison representation, whether obtained through inspect or from a designated state drive. -- `delete_dump(prefix)` removes the app-owned checkpoint. The sequencer owns - the outer directory and `info.toml`; the app owns its opaque `state` prefix. +- All checkpoint-owned artifacts reside at or below `prefix`, including the + canonical comparison file. Disposal requires only ordinary filesystem + deletion: the sequencer removes the enclosing directory, including its own + `info.toml`. Removing a checkpoint must leave other checkpoints and restored + engines usable; external resource cleanup is outside this contract. Mutable checkpoint creation permits flushing, replacing mappings, or changing working backing files inside an adapter. It does not permit a logical state @@ -185,6 +190,9 @@ also stays on the concrete application. 4. Implement `CanonicalState` only where canonical inspection needs it. Remove any `Clone` or `Sync` added solely to satisfy the old host bounds; justify `Send` against the native engine's ownership contract. +5. Return the stable payload bound from `max_method_payload_bytes()` rather + than an associated constant. Keep checkpoint artifacts self-contained so + recursive filesystem deletion disposes of them without an application hook. Changing these Rust interfaces preserves transaction encoding, expected rejection semantics, snapshot bytes, scheduler ordering, and the database diff --git a/docs/protocol/c-application-binding.md b/docs/protocol/c-application-binding.md new file mode 100644 index 00000000..739be090 --- /dev/null +++ b/docs/protocol/c-application-binding.md @@ -0,0 +1,49 @@ +# C application binding + +An application exposes the +[`application-engine.h`](../../examples/c-app-engine/include/application-engine.h) +C ABI in a static archive. `c-app-engine::EngineApp` adapts that engine to the +Rust `Application` trait. The [build guide](../../examples/c-app-engine/README.md) +covers linking, genesis, host commands, and reference tests. + +## Contract ownership + +- The [Application contract](application-contract.md) defines execution, + rejection, progress, deterministic outputs, and checkpoint obligations. +- [Scheduler semantics](scheduler-semantics.md) defines canonical ordering and + the acceptance boundary. The native engine does not acquire an ordering role + through this ABI. +- The [C header](../../examples/c-app-engine/include/application-engine.h) + defines record layout, statuses, pointer ownership, and call lifetimes. An + engine archive and its generated Rust bindings must agree on that header. +- [Snapshot lifecycle](../snapshots/lifecycle.md) owns checkpoint registration, + promotion, reader leases, and garbage collection. The application owns its + checkpoint representation within the supplied prefix. + +## Execution and ownership + +Each opaque engine handle owns its mutable state. It may move between threads, +and calls on that handle are exclusive. Progress is returned together as +`ApplicationEngineProgress`; the shared execution boundary checks it against the +successful transition. The payload-bound getter takes no instance and returns a +stable value for the linked engine. Zero is a valid bound. + +Expected validation rejection leaves state unchanged. An execution, validation, +or output-drain failure discards the engine; the host owns process termination +and distinguishes terminal faults from retryable operational failures. +Exceptions must not cross the ABI. The header defines the lifetimes of borrowed +output and diagnostic buffers; the adapter copies them before reuse. + +Fee fields carry base-129/128 exponents. The shared max-fee comparison operates +in log space; an application checking balances or charging fees uses a linear +amount. The reference conversion lives in +[`sequencer-core/src/fee.rs`](../../sequencer-core/src/fee.rs). Native and +canonical execution must agree on that conversion, since different amounts can +change rejection decisions and resulting balances. + +The same implementation may be compiled for native execution and the canonical +machine. This does not establish equivalent behavior across targets: the +application must preserve deterministic state and output bytes, including the +checkpoint's canonical comparison file. Reference wallet ABI tests exercise +the host integration; they do not establish private DEX conformance or +equivalence between native and machine execution. diff --git a/docs/review/register.md b/docs/review/register.md index 88fa030f..c30330ae 100644 --- a/docs/review/register.md +++ b/docs/review/register.md @@ -208,6 +208,13 @@ Open maintainer decisions: seed test digests 5,000 directs over a 7,200-block jump in one turn). - **Track 6 with Bart**: the hardlink-suitability dispute and the changed-era bootstrap contract (see the tracks doc). +- **C engine follow-up boundary** (2026-09-11): agree output storage and + checkpoint layout with Bart before replacing the drain protocol with output + arrays or the path callback with a static suffix. ABI version negotiation, + checked-in bindings, and linker policy also need concrete integration + requirements. Keep these separate from the port and the agreed contract + reductions; the private DEX engine and scheduler remain unavailable for + conformance testing. ## Owed tests @@ -271,6 +278,12 @@ Statuses swept 2026-08-22 and updated through 2026-09-04. (unit-level provider mocks instead; e2e validates only the passing path) and fsync/power-loss WAL-rewind injection (out of scope — state-construction variants cover the detectable halves). +- **External-engine conformance** (2026-09-11): a reusable runner needs + application-supplied genesis and meaningful accepted/rejected inputs; compare + canonical comparison files rather than byte-identical recovery dumps. + Native repeatability alone does not establish native/canonical-machine + equivalence. Publish fee-conversion data and vectors for the C++ integration + (finding 8), and add targeted devnet E2E coverage through the C host in CI. ## Settled decisions @@ -342,6 +355,24 @@ Each entry: the decision, its reason, and where the reasoning now lives. directory of state files. GC racing a download is an ordinary supported operation; relying on open-file unlink behavior is not the general dump lifetime contract → snapshot lifecycle, leases. +- **C bridge contract reductions** (2026-09-11): the linked engine reports its + stable payload bound at runtime, including a valid zero bound; C progress is + one count/clock record. This removes duplicated build configuration and + presents the protocol pair together without introducing another state owner. + Checkpoint artifacts are self-contained beneath their supplied prefix and + require only ordinary filesystem deletion, so the sequencer recursively + removes its enclosing dump directory without an application deletion hook. + Reader leases, SQLite-first deletion, and restored-engine independence remain + required → [C binding guide](../protocol/c-application-binding.md) and + [Application checkpoint contract](../protocol/application-contract.md#6-checkpoint-lifecycle). +- **C host failure boundaries** (2026-09-11): snapshot-path callback panics + enter the host's terminal-abort boundary; a failed HTTP task alone must not + leave sequencing active after an application invariant violation. Missing or + malformed genesis dumps and absent required options are terminal bootstrap + failures; operational I/O remains retryable. Genesis stays lazy: completed setup may + succeed after its original source has been deleted. A pre-command mandatory + state-file check would violate that lifecycle contract → snapshot handlers, + command error classification, and genesis harness tests. - **Execution-offset continuity has one enforcement point** (2026-09-07): the SQLite trigger rejects a noncanonical offset inside the physical-row transaction. The duplicate Rust loop was removed; rollback, invalidation, @@ -668,3 +699,4 @@ for `2026-06-10-correctness-review.md`, `2026-06-10-simplification.md`, | 2026-09-03 | Branch stock-take of the authority-boundary PR: first-hand reads, then a read-only fleet of seven subsystem lenses and five premise challengers, then three refuters over the eighteen highest-ranked proposals | Proportionate overall, with three residue pockets; every proposal recorded, the jury-refuted ones listed above | [`2026-09-03-branch-stocktake.md`](2026-09-03-branch-stocktake.md), the ledger of the current branch, with its "Landed" section | | 2026-09-07 | PR #28 premise review and maintainer-approved simplification | Ordered recovery replaces the phase driver; diagnosed terminal runtime faults abort immediately; ordinary shutdown and snapshot leases remain; reader drain race and duplicate offset check fixed | Current ADR and recovery design; finding 33 and settled decisions above. Validation: 692 host tests, seven targeted restart/outage E2Es, workspace check, strict Clippy, formatting, and admission TLC passed. The broader stale-batch recovery E2E reached its watchdog comparison but was blocked by the host Lua emulator 0.21 loading the pinned 0.20 image (archive version mismatch); no protocol pin was changed. | | 2026-09-09 | Application, inclusion lane, and public DEX integration branch | Native progress ownership, typed validation failures, mutable independent checkpoints, optional canonical inspection, and lane bookkeeping simplified; ingress CORS and Lua 5.4 parity restored. Reference C bridge port kept separate. | [Application/lane review](2026-09-09-application-lane-dex-review.md); current Application and snapshot contracts. Workspace check, strict Clippy, 697 host tests, and 62 watchdog tests passed; private DEX conformance remains unverified. | +| 2026-09-11 | Reference C bridge port and review boundary | Keep the current Application contract, runtime payload bound, paired progress, filesystem-owned checkpoint disposal, and host failure fixes together. Engine-dependent API refinements and external-engine conformance remain follow-ups. | Settled decisions and owed tests above; [C binding guide](../protocol/c-application-binding.md), Application contract, and snapshot lifecycle. | diff --git a/docs/snapshots/README.md b/docs/snapshots/README.md index fba2de05..48e8e7ed 100644 --- a/docs/snapshots/README.md +++ b/docs/snapshots/README.md @@ -8,7 +8,7 @@ operator's watchdog (`/finalized_state`) and indexers (`/latest_snapshot`). Two documents, split by concern: - **[`format.md`](format.md)** — the on-disk *format*: the `Application` dump - trait (`from_dump` / `create_dump` / `delete_dump` / `state_file_in_dump`) and + trait (`from_dump` / `create_dump` / `state_file_in_dump`) and the toy wallet's SSZ wire encoding. What a dump *is*. - **[`lifecycle.md`](lifecycle.md)** — the *lifecycle* and its rationale: take at diff --git a/docs/snapshots/format.md b/docs/snapshots/format.md index f07cc935..cdf04eb9 100644 --- a/docs/snapshots/format.md +++ b/docs/snapshots/format.md @@ -9,7 +9,7 @@ This document covers two things: 1. The trait shape that any `Application` implementation must satisfy to participate in snapshot lifecycle (`from_dump`, `create_dump`, - `delete_dump`, `state_file_in_dump`). + `state_file_in_dump`). 2. The wire format the toy wallet uses to encode its canonical state into the dump's state file. Checkpoint ownership and durability are defined by the [Application contract](../protocol/application-contract.md#6-checkpoint-lifecycle). @@ -27,7 +27,6 @@ trait Application: Send + Sized { fn from_dump(prefix: &Path) -> Result; fn create_dump(&mut self, prefix: &Path) -> Result<(), AppError>; - fn delete_dump(prefix: &Path) -> Result<(), AppError>; fn state_file_in_dump(prefix: &Path) -> PathBuf; } ``` @@ -35,7 +34,9 @@ trait Application: Send + Sized { Contract: - `prefix` is an opaque app-owned path, which may be a file or directory. - The sequencer owns the enclosing dump directory and its `info.toml`. + All checkpoint-owned artifacts reside at or below it. The sequencer owns the + enclosing dump directory and its `info.toml`, and disposes of the checkpoint + by removing that directory recursively. No other resource cleanup is needed. - `create_dump` creates the absent `prefix` and makes the complete checkpoint durable before returning. It may replace backing resources but preserves logical state. Later execution cannot alter a checkpoint. Restored engines @@ -108,7 +109,7 @@ and its canonical state coincide; one write per `create_dump`. - `nonce` (`u32`) - `executed_input_count` (`u64`) - `last_executed_safe_block` (`u64`) — the app's safe-block clock - (`Application::last_executed_safe_block`): max block carried by any + (reported by `Application::progress`): max block carried by any executed input. Recovery reads it as `A`, the safe block this state reflects, so it must live in the canonical state bytes (both the bare-metal and canonical-machine sides advance it identically). diff --git a/docs/snapshots/lifecycle.md b/docs/snapshots/lifecycle.md index 8ba9036d..45654e48 100644 --- a/docs/snapshots/lifecycle.md +++ b/docs/snapshots/lifecycle.md @@ -105,7 +105,7 @@ Three SQLite tables back it (`storage/migrations/0001_schema.sql`): ```text dumps// state app-owned file or directory — the prefix handed to - Application::{create_dump, from_dump, delete_dump} + Application::{create_dump, from_dump} (opaque to the sequencer; see format.md) info.toml sequencer-owned checkpoint metadata: format_version, next_batch_nonce (N), l2_tx_index, @@ -364,8 +364,9 @@ gives the create/delete orderings: the reference row is written. - **SQLite delete → file delete** (SQLite-first): `gc_unreferenced_dumps` deletes the rows inside one `write` tx and *returns* the prefixes; the lane's - `run_gc` then `A::delete_dump`s them after commit. If a directory delete - fails, an orphan file is acceptable — the next startup's `sweep_orphan_dumps` + `run_gc` then removes the enclosing dump directories recursively after + commit. If a directory delete fails, an orphan file is acceptable — the next + startup's `sweep_orphan_dumps` catches it. The reverse ordering would leave a dangling row. This is why `storage/snapshot_dumps.rs` is SQLite-only and the FS half lives in diff --git a/examples/app-core/src/application/wallet.rs b/examples/app-core/src/application/wallet.rs index 8e08c4e4..db7258f2 100644 --- a/examples/app-core/src/application/wallet.rs +++ b/examples/app-core/src/application/wallet.rs @@ -219,7 +219,9 @@ impl Default for WalletApp { } impl Application for WalletApp { - const MAX_METHOD_PAYLOAD_BYTES: usize = WALLET_MAX_METHOD_PAYLOAD_BYTES; + fn max_method_payload_bytes() -> usize { + WALLET_MAX_METHOD_PAYLOAD_BYTES + } fn validate_user_op( &self, @@ -383,11 +385,6 @@ impl Application for WalletApp { Ok(()) } - fn delete_dump(prefix: &Path) -> Result<(), AppError> { - std::fs::remove_dir_all(prefix)?; - Ok(()) - } - fn state_file_in_dump(prefix: &Path) -> PathBuf { prefix.join("state") } @@ -869,7 +866,7 @@ mod tests { let restored = WalletApp::from_dump(&prefix).expect("load dump"); - WalletApp::delete_dump(&prefix).expect("cleanup dump"); + std::fs::remove_dir_all(&prefix).expect("cleanup dump"); assert_eq!( restored.config.erc20_portal_address, @@ -919,7 +916,7 @@ mod tests { std::fs::read(WalletApp::state_file_in_dump(&prefix)).unwrap(), before ); - WalletApp::delete_dump(&prefix).unwrap(); + std::fs::remove_dir_all(&prefix).unwrap(); execute_valid_user_op(&mut first, &user_op, 13).unwrap(); assert_eq!(first.executed_input_count().get(), 2); } @@ -994,8 +991,8 @@ mod tests { let bytes_a = std::fs::read(WalletApp::state_file_in_dump(&prefix_a)).expect("read a"); let bytes_b = std::fs::read(WalletApp::state_file_in_dump(&prefix_b)).expect("read b"); - WalletApp::delete_dump(&prefix_a).expect("cleanup a"); - WalletApp::delete_dump(&prefix_b).expect("cleanup b"); + std::fs::remove_dir_all(&prefix_a).expect("cleanup a"); + std::fs::remove_dir_all(&prefix_b).expect("cleanup b"); assert_eq!( bytes_a, bytes_b, @@ -1011,7 +1008,7 @@ mod tests { .expect("write malformed"); let err = WalletApp::from_dump(&prefix).expect_err("invalid bytes should fail"); - WalletApp::delete_dump(&prefix).expect("cleanup"); + std::fs::remove_dir_all(&prefix).expect("cleanup"); match err { AppError::Internal { reason } => assert!( diff --git a/examples/c-app-engine/README.md b/examples/c-app-engine/README.md index 705299c9..254ec262 100644 --- a/examples/c-app-engine/README.md +++ b/examples/c-app-engine/README.md @@ -4,6 +4,8 @@ [application-engine C ABI](include/application-engine.h). The sequencer owns one engine at a time. A handle can move between threads; calls on it never overlap. The bridge is `Send`, without `Clone` or `Sync`. +The [binding guide](../../docs/protocol/c-application-binding.md) maps the ABI +to the Application and scheduler contracts. The native engine owns application state and its execution count/safe-block clock. Successful execution advances that progress, including counted no-ops; @@ -21,6 +23,9 @@ A dump prefix may be a file or a directory. Opening it produces independently mutable state without changing the source; checkpoint creation may mutate the engine's backing arrangement, while preserving logical state and progress. Successful checkpoints are durable and immutable under subsequent execution. +All checkpoint artifacts reside at or below the prefix; the sequencer disposes +of them with ordinary recursive filesystem deletion. Restored engines remain +usable after source deletion. `state_file_in_dump` names the one canonical comparison file, which can be the whole dump or a projection alongside richer restoration artifacts. `EngineApp` does not implement the optional Rust `CanonicalState` inspection trait. @@ -47,8 +52,9 @@ The host adds `--state-file` through its own parser and passes the parsed comman to `sequencer::run_command`. This shares `run_main`'s command lifecycle and exit policy. Both take a lazy `FnOnce() -> Result` genesis factory; infallible Rust constructors therefore use -`run_main(|| Ok(WalletApp::new(WalletConfig::default())))`. A missing genesis file -returns an ordinary application-bootstrap I/O error, while a caught factory +`run_main(|| Ok(WalletApp::new(WalletConfig::default())))`. An absent required +genesis path or a missing/corrupt genesis dump is a terminal bootstrap error; +operational I/O failures retain their retryable classification. A caught factory panic follows the shared terminal-error policy. ## External engine @@ -58,14 +64,15 @@ Build the application's static archive and use the corresponding header: ```sh APPLICATION_ENGINE_LIB=/absolute/path/libengine.a \ APPLICATION_ENGINE_HEADER=/absolute/path/application-engine.h \ -APPLICATION_ENGINE_METHOD_PAYLOAD_LIMIT=53 \ cargo build -p c-app-sequencer ``` -The payload limit must match the engine's own build. Bindgen generates the Rust -records from that header, so a build needs libclang. The engine also supplies its -own genesis tool; configuration does not cross this ABI. With no external -archive configured, the generic binary reports that no engine was linked, and +The linked engine reports its stable payload bound through +`application_engine_max_method_payload_bytes()`; zero permits only empty method +payloads. Bindgen generates the Rust records from the header, so a build needs +libclang. The engine also supplies its own genesis tool; configuration does not +cross this ABI. With no external archive configured, the generic binary reports +that no engine was linked, and `c-wallet-sequencer` supplies the reference implementation through Cargo. The conformance suite compares native and ABI execution over mixed inputs, diff --git a/examples/c-app-engine/build.rs b/examples/c-app-engine/build.rs index b7b9e677..1307dcbf 100644 --- a/examples/c-app-engine/build.rs +++ b/examples/c-app-engine/build.rs @@ -3,27 +3,17 @@ //! Links the application's engine archive and generates the FFI declarations from its header. //! -//! The three environment variables are the whole application-specific binding, documented in +//! The two environment variables are the whole application-specific binding, documented in //! `README.md`. With none of them set this crate links no archive, //! and the binary that uses it supplies the engine instead. use std::env; use std::path::{Path, PathBuf}; -/// The in-workspace wallet engine's own bound, used when a build declares none. -/// -/// Only reachable when `c-wallet-engine` is the engine, which asserts this same value against -/// `WalletApp::MAX_METHOD_PAYLOAD_BYTES`, so a number that drifts fails that crate's compile -/// rather than reaching a host. An application outside this workspace always declares its own. -const REFERENCE_ENGINE_METHOD_PAYLOAD_LIMIT: u32 = 1 + 32 + 20; - /// Generate `sys`'s contents from the engine header, the one authoritative declaration of what /// the archive exports, so a change on the engine side is either picked up here or fails this /// build. -/// -/// The payload bound is defined for the parse rather than read out of the header, because the -/// header deliberately refuses to carry a default for it. -fn generate_bindings(header: &Path, method_payload_limit: u32) { +fn generate_bindings(header: &Path) { let out_path = PathBuf::from(env::var("OUT_DIR").expect("OUT_DIR")).join("bindings.rs"); let bindings = bindgen::Builder::default() .header( @@ -34,9 +24,6 @@ fn generate_bindings(header: &Path, method_payload_limit: u32) { // Parse the C arm of the header. Its C++ arm only spells noexcept, which has no bearing // on the ABI and no Rust spelling. .clang_args(["-x", "c", "-std=c11"]) - .clang_arg(format!( - "-DAPPLICATION_ENGINE_METHOD_PAYLOAD_LIMIT={method_payload_limit}" - )) // Only the seam's own surface, never what stdint.h drags in behind it .allowlist_item("^(application_engine_|ApplicationEngine|APPLICATION_ENGINE_).*") // Plain integer constants, never Rust enums. The contract requires refusing a value the @@ -98,10 +85,8 @@ fn link_application_archive(engine_lib: &Path) { /// What an application supplying its own archive has to declare alongside it. /// -/// Both are demanded rather than defaulted. The archive and the header are separate artifacts and -/// only that pairing is meaningful, and a bound guessed here would be exactly the silently wrong -/// number the header's own `#error` exists to prevent. -fn external_engine(engine_lib: &str) -> (PathBuf, u32) { +/// The archive and header must come from the same engine build. +fn external_engine(engine_lib: &str) -> PathBuf { link_application_archive(Path::new(engine_lib)); let header = PathBuf::from(env::var("APPLICATION_ENGINE_HEADER").expect( @@ -113,38 +98,26 @@ fn external_engine(engine_lib: &str) -> (PathBuf, u32) { header.display() ); - let declared = env::var("APPLICATION_ENGINE_METHOD_PAYLOAD_LIMIT").expect( - "APPLICATION_ENGINE_METHOD_PAYLOAD_LIMIT is unset, set it to the application's largest \ - method payload, the same value the archive was built with", - ); - let limit = declared.trim().parse::().unwrap_or_else(|err| { - panic!("APPLICATION_ENGINE_METHOD_PAYLOAD_LIMIT is not a number: {err}") - }); - (header, limit) + header } fn main() { println!("cargo::rerun-if-env-changed=APPLICATION_ENGINE_LIB"); println!("cargo::rerun-if-env-changed=APPLICATION_ENGINE_HEADER"); - println!("cargo::rerun-if-env-changed=APPLICATION_ENGINE_METHOD_PAYLOAD_LIMIT"); let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR")); - let (header, method_payload_limit) = match env::var("APPLICATION_ENGINE_LIB") { + let header = match env::var("APPLICATION_ENGINE_LIB") { Ok(engine_lib) => external_engine(&engine_lib), // Linked from inside this workspace, where `c-wallet-engine` is the engine Err(_) => { assert!( - env::var_os("APPLICATION_ENGINE_HEADER").is_none() - && env::var_os("APPLICATION_ENGINE_METHOD_PAYLOAD_LIMIT").is_none(), + env::var_os("APPLICATION_ENGINE_HEADER").is_none(), "external engine settings require APPLICATION_ENGINE_LIB" ); - ( - manifest_dir.join("include").join("application-engine.h"), - REFERENCE_ENGINE_METHOD_PAYLOAD_LIMIT, - ) + manifest_dir.join("include").join("application-engine.h") } }; println!("cargo::rerun-if-changed={}", header.display()); - generate_bindings(&header, method_payload_limit); + generate_bindings(&header); } diff --git a/examples/c-app-engine/include/application-engine.h b/examples/c-app-engine/include/application-engine.h index 6adeb81f..15562c62 100644 --- a/examples/c-app-engine/include/application-engine.h +++ b/examples/c-app-engine/include/application-engine.h @@ -21,9 +21,9 @@ /// deployment, so a host never learns what configures the application it runs, and the path is /// opaque, a file or a directory as the engine chooses. /// -/// This header is the surface a host binds to, and the Rust host generates its declarations from -/// it with bindgen rather than restating them, so a signature changed here cannot disagree with -/// the host that links the engine. The records that cross are plain C layout and the +/// The Rust host generates its declarations from this header with bindgen. The supplied header +/// must match the linked archive; binding generation does not verify that pairing. +/// The records that cross are plain C layout and the /// engine must static_assert their sizes and field offsets, so a compiler laying one out /// differently fails its build rather than the seam. A generated binding carries the same checks /// on the host side. @@ -44,8 +44,17 @@ /// /// No exception may cross. Fallible entry points report errors through status codes; a fatal /// validation or execution error defines no successor and the caller must discard the instance. -/// Lifecycle statuses distinguish operational I/O from missing or malformed dump artifacts. -/// Only accept or reject is consensus visible; rejection diagnostics are descriptive. +/// IO_ERROR is legal from every status-returning entry point. In validation, execution, and +/// output draining it is fatal for the instance; the host may retry with a fresh instance after +/// an operational failure. NOT_FOUND and INVALID_DUMP describe lifecycle failures only. +/// Validation acceptance, state transitions, progress, and outputs must agree with the canonical +/// application. Rejection diagnostics and error messages need not be identical across builds. +/// +/// Fees are uint16_t exponents with base 129/128, denominated in the fee token's smallest unit. +/// The conversion is defined by sequencer-core/src/fee.rs and its build.rs-generated table: +/// integer fixed-point arithmetic with 64 fractional bits, including its rounding and exponent +/// bound. Native and canonical implementations must agree on this conversion; the max-fee guard +/// compares exponents, while balance validation and execution use the converted amount. /// /// Errors are errno style. A fallible entry point returns a status (or a null pointer for /// application_engine_state_file_in_dump) and leaves the reason for @@ -79,34 +88,15 @@ /// raw big-endian bytes rather than as a number a host may not be able to spell. #define APPLICATION_ENGINE_VALUE_SIZE 32 -/// @brief The one value this header does not fix, supplied by the application's build. -/// @details The ingress bound on a single user op's method payload is an application sizing -/// decision, the largest payload any of its methods can carry, so it is defined on the compile -/// line rather than here. Every consumer of this header, the engine's own translation units and -/// the binding generation alike, must be given the same value, which is what keeps the bound the -/// host enforces and the bound the engine parses under from being two numbers. -/// -/// There is deliberately no default. A silently wrong bound is the exact failure this -/// declaration exists to prevent, so an undefined one stops the build here. -#ifndef APPLICATION_ENGINE_METHOD_PAYLOAD_LIMIT -#error "define APPLICATION_ENGINE_METHOD_PAYLOAD_LIMIT to the application's largest method payload" -#endif - #ifdef __cplusplus extern "C" { #endif -/// @brief The bounds an engine declares, spelled as constants a generated binding can read. -/// @details A binding generator sees a macro only where it is defined, and -/// APPLICATION_ENGINE_METHOD_PAYLOAD_LIMIT is defined on the compile line, so the value is -/// restated here as an enumeration constant. That is what carries it across to a host: the -/// application defines one number, and both sides read it from this declaration. -typedef enum ApplicationEngineLimits { - /// The largest method payload a user op may carry, in bytes. The host publishes it as the - /// sequencer's MAX_METHOD_PAYLOAD_BYTES and refuses anything larger, so an engine that raised - /// its own bound without raising this one would never see the payloads it grew to accept. - APPLICATION_ENGINE_MAX_METHOD_PAYLOAD_BYTES = APPLICATION_ENGINE_METHOD_PAYLOAD_LIMIT, -} ApplicationEngineLimits; +/// @brief The largest user-op method payload, in bytes. +/// @details Pure, infallible, and constant for the linked engine implementation, independent of +/// any loaded deployment. Zero permits only empty method payloads. The host uses this value for +/// ingress admission and batch sizing; it does not bound direct inputs from L1. +APPLICATION_ENGINE_API uint64_t application_engine_max_method_payload_bytes(void) APPLICATION_ENGINE_NOEXCEPT; /// @brief An account address, raw bytes and no encoding. /// @details A named type rather than a loose buffer, so an address and an amount cannot be @@ -147,7 +137,7 @@ typedef enum ApplicationEngineStatus { /// that guard belongs to the caller, so an op arrives whole. typedef struct ApplicationEngineUserOp { uint32_t nonce; ///< Sender replay protection nonce. - uint16_t max_fee; ///< Highest frame fee price the sender accepts, in log space. + uint16_t max_fee; ///< Highest frame fee exponent the sender accepts, base 129/128. ApplicationEngineByteSpan data; ///< Method payload, opaque here and parsed by the engine. } ApplicationEngineUserOp; @@ -156,13 +146,13 @@ typedef struct ApplicationEngineUserOp { /// went with the guard the caller already settled, leaving the fee the frame charges. typedef struct ApplicationEngineValidUserOp { ApplicationEngineEthereumAddress sender; ///< The recovered signer. - uint16_t fee; ///< The frame fee price charged, in log space. + uint16_t fee; ///< The charged frame fee exponent, base 129/128. ApplicationEngineByteSpan data; ///< Method payload, opaque here and parsed by the engine. } ApplicationEngineValidUserOp; /// @brief An input taken straight from the L1 input box. -/// @details Its sender is authenticated by the chain rather than recovered from a signature, -/// which is what lets the engine trust it without validating anything first. +/// @details Its sender is authenticated by the chain rather than recovered from a signature. +/// Its payload is still untrusted application input and has no method-payload bound at this ABI. typedef struct ApplicationEngineDirectInput { ApplicationEngineEthereumAddress sender; ///< The L1 authenticated sender. uint64_t block_number; ///< The L1 inclusion block number. @@ -248,6 +238,12 @@ typedef struct ApplicationEngineOutput { /// @brief The engine instance behind the handle, opaque to every caller. typedef struct ApplicationEngine ApplicationEngine; +/// @brief Progress embedded in the engine's logical state and every checkpoint. +typedef struct ApplicationEngineProgress { + uint64_t executed_input_count; ///< Next input offset; included no-ops count, rejections do not. + uint64_t last_executed_safe_block; ///< Maximum block carried by any executed input. +} ApplicationEngineProgress; + /// @brief Get the message describing the most recent failure. /// @returns A NUL terminated string, never null, empty when the last fallible call succeeded. /// @details Read it after a negative status or a null handle. Every fallible entry point clears @@ -289,12 +285,12 @@ APPLICATION_ENGINE_API void application_engine_destroy(ApplicationEngine *engine /// @param engine The engine handle. /// @param sender The recovered signer. /// @param user_op The op to validate, as its sender signed it. -/// @param current_fee The frame fee price in log space. +/// @param current_fee The frame fee exponent, base 129/128. /// @param out_invalid Why the op was refused, written whole and only on INVALID. -/// @returns OK, INVALID with diagnostics, or INTERNAL_ERROR. +/// @returns OK, INVALID with diagnostics, IO_ERROR, or INTERNAL_ERROR. /// @details A rejection reports itself through out_invalid and leaves the last error message -/// empty, only INTERNAL_ERROR carries one. The max-fee guard belongs to the caller and is never -/// checked here, so APPLICATION_ENGINE_INVALID_MAX_FEE never comes back from this call. Queued +/// empty; IO_ERROR and INTERNAL_ERROR carry an error message. The max-fee guard belongs to the +/// caller and is never checked here, so APPLICATION_ENGINE_INVALID_MAX_FEE never comes back. Queued /// outputs are left alone, only an execution touches them. APPLICATION_ENGINE_API ApplicationEngineStatus application_engine_validate_user_op(const ApplicationEngine *engine, const ApplicationEngineEthereumAddress *sender, const ApplicationEngineUserOp *user_op, uint16_t current_fee, @@ -305,10 +301,10 @@ APPLICATION_ENGINE_API ApplicationEngineStatus application_engine_validate_user_ /// @param user_op The validated op to execute. /// @param safe_block The covering frame safe block, folded into the clock as max(clock, it). /// @param out_output_count How many outputs this op left waiting, written only on OK. -/// @returns OK or INTERNAL_ERROR (an engine throw is fatal-no-resume). +/// @returns OK, IO_ERROR, or INTERNAL_ERROR. An error requires discarding the instance. /// @details An op the method rejects still executed and still counts, so it reports OK. Only -/// accept or reject is consensus visible and the state carries it, the seam does not surface -/// the application's own reason. +/// the resulting state, progress, and outputs cross the seam; the application's business-failure +/// diagnostics are not returned separately. /// /// An execution refuses to run while an earlier execution's outputs are still queued, reporting /// INTERNAL_ERROR without executing anything rather than discarding outputs meant to reach the @@ -321,7 +317,7 @@ APPLICATION_ENGINE_API ApplicationEngineStatus application_engine_execute_valid_ /// @param engine The engine handle. /// @param input The input to execute, its L1 block folded into the clock as max(clock, it). /// @param out_output_count How many outputs this input left waiting, written only on OK. -/// @returns OK or INTERNAL_ERROR (an engine throw is fatal-no-resume). +/// @returns OK, IO_ERROR, or INTERNAL_ERROR. An error requires discarding the instance. /// @details An input the engine rejects is a counted no-op and still reports OK, the same way a /// rejected user op does. Outputs behave as they do for a user op. APPLICATION_ENGINE_API ApplicationEngineStatus application_engine_execute_direct_input(ApplicationEngine *engine, @@ -330,7 +326,7 @@ APPLICATION_ENGINE_API ApplicationEngineStatus application_engine_execute_direct /// @brief Take the next queued output, in emission order. /// @param engine The engine handle. /// @param out_output The output taken, written whole and only on OK. -/// @returns OK with an output written, or INTERNAL_ERROR. +/// @returns OK with an output written, IO_ERROR, or INTERNAL_ERROR. /// @details Call it exactly as many times as the execution reported, which is what attributes /// the outputs to the input that produced them. Taking one more than were queued is a caller bug /// and reports INTERNAL_ERROR rather than an empty output a host might act on. The payload @@ -340,47 +336,39 @@ APPLICATION_ENGINE_API ApplicationEngineStatus application_engine_execute_direct APPLICATION_ENGINE_API ApplicationEngineStatus application_engine_drain_output(ApplicationEngine *engine, ApplicationEngineOutput *out_output) APPLICATION_ENGINE_NOEXCEPT; -/// @brief Get the maximum block carried by any executed input (the engine's safe-block clock). -/// @param engine The engine handle. -/// @returns The last executed safe block, zero when nothing has executed. -/// @details Carried by execution rather than set, so an engine cannot execute and forget to -/// advance it. It lives in the state, so a resumed one reports the block it reflects. -APPLICATION_ENGINE_API uint64_t application_engine_last_executed_safe_block( - const ApplicationEngine *engine) APPLICATION_ENGINE_NOEXCEPT; - -/// @brief Get the count of executed inputs, user ops and direct inputs alike. +/// @brief Read the engine's current progress without changing state. /// @param engine The engine handle. -/// @returns The executed input count. -APPLICATION_ENGINE_API uint64_t application_engine_executed_input_count( - const ApplicationEngine *engine) APPLICATION_ENGINE_NOEXCEPT; +/// @param out_progress Writable record, written whole before returning. +/// @details Count zero implies clock zero. Every successful execution advances the count once +/// with checked arithmetic and takes max(previous clock, input block). Both fields survive dumps. +APPLICATION_ENGINE_API void application_engine_progress(const ApplicationEngine *engine, + ApplicationEngineProgress *out_progress) APPLICATION_ENGINE_NOEXCEPT; /// @brief Create a crash durable dump of the engine state (write, fsync). /// @param engine The engine handle. /// @param prefix The dump to create, must not pre-exist. It carries whatever shape the engine's /// state does, a directory or a plain file as the engine chooses. -/// @returns OK, IO_ERROR when the filesystem refused, which is what a full filesystem or an -/// exhausted quota reports, or INTERNAL_ERROR. +/// @returns OK, NOT_FOUND for a missing required path, IO_ERROR for other filesystem failures, +/// or INTERNAL_ERROR. /// @details Must be called at a quiescent point only, no in-flight execution. On OK the dump /// survives an immediate kernel crash, its payload and the directory entry naming it are both -/// synchronized before returning. An engine that cleans up after a failed write leaves the prefix -/// free for a clean retry, which a host cannot do on its behalf. +/// synchronized before returning. Failed creation may leave artifacts beneath prefix for the +/// host to remove with the unreferenced checkpoint directory. /// /// The dump contains the current state. The engine may change backing files or reopen internal /// handles while checkpointing, but application state and progress stay unchanged. Subsequent /// execution must leave the completed dump immutable. +/// All checkpoint-owned artifacts reside at or beneath prefix as ordinary files/directories. +/// The host discards a checkpoint by recursive filesystem deletion, with no engine callback. +/// Deletion must leave other checkpoints and independently restored engines usable. Filesystem +/// CoW sharing is allowed when writes remain isolated. +/// +/// The file named by application_engine_state_file_in_dump must byte-equal the canonical build's +/// deterministic inspection or designated state-drive representation for the same logical state. +/// The bridge does not verify this cross-build equivalence. APPLICATION_ENGINE_API ApplicationEngineStatus application_engine_create_dump(ApplicationEngine *engine, const char *prefix) APPLICATION_ENGINE_NOEXCEPT; -/// @brief Delete a previously created dump. -/// @param prefix The dump to remove. -/// @returns OK, NOT_FOUND when the dump is absent, IO_ERROR for other filesystem failures, -/// or INTERNAL_ERROR. -/// @details An engine still holding this dump open keeps running, its mapping outlives the name. -/// The sequencer removes the database reference before deleting the artifact. Deletion must not -/// affect other dumps or independently loaded engines. -APPLICATION_ENGINE_API ApplicationEngineStatus application_engine_delete_dump( - const char *prefix) APPLICATION_ENGINE_NOEXCEPT; - /// @brief Get the path of the canonical state file inside a dump. /// @param prefix The dump to name the state file of. /// @returns A NUL terminated path, or null on failure with the reason in the last error message. @@ -388,6 +376,8 @@ APPLICATION_ENGINE_API ApplicationEngineStatus application_engine_delete_dump( /// file sits follows from the shape the engine gives a dump, which is why the engine answers /// rather than a host assuming. An engine whose dump is a directory answers with a file inside /// it, and one whose dump is the state image itself answers with the prefix unchanged. +/// Its bytes must match the independent canonical build's deterministic comparison representation +/// for the same logical state, as required by application_engine_create_dump. /// /// The storage is engine owned and thread local, overwritten by the next call on the same /// thread, so copy rather than retain the pointer. Being fallible, it also clears the last error diff --git a/examples/c-app-engine/src/lib.rs b/examples/c-app-engine/src/lib.rs index 24a4d223..2ec2f35a 100644 --- a/examples/c-app-engine/src/lib.rs +++ b/examples/c-app-engine/src/lib.rs @@ -137,8 +137,10 @@ impl EngineApp { } impl Application for EngineApp { - const MAX_METHOD_PAYLOAD_BYTES: usize = - sys::APPLICATION_ENGINE_MAX_METHOD_PAYLOAD_BYTES as usize; + fn max_method_payload_bytes() -> usize { + usize::try_from(unsafe { sys::application_engine_max_method_payload_bytes() }) + .expect("engine payload bound exceeds usize") + } fn validate_user_op( &self, @@ -240,10 +242,16 @@ impl Application for EngineApp { } fn progress(&self) -> ApplicationProgress { - let count = unsafe { sys::application_engine_executed_input_count(self.engine) }; - let clock = unsafe { sys::application_engine_last_executed_safe_block(self.engine) }; - ApplicationProgress::try_new(ExecutedInputCount::new(count), clock) - .expect("engine returned incoherent application progress") + let mut progress = sys::ApplicationEngineProgress { + executed_input_count: 0, + last_executed_safe_block: 0, + }; + unsafe { sys::application_engine_progress(self.engine, &mut progress) }; + ApplicationProgress::try_new( + ExecutedInputCount::new(progress.executed_input_count), + progress.last_executed_safe_block, + ) + .expect("engine returned incoherent application progress") } fn from_dump(prefix: &Path) -> Result { @@ -268,14 +276,6 @@ impl Application for EngineApp { ) } - fn delete_dump(prefix: &Path) -> Result<(), AppError> { - let prefix = path_to_cstring(prefix); - check( - unsafe { sys::application_engine_delete_dump(prefix.as_ptr()) }, - "delete_dump", - ) - } - fn state_file_in_dump(prefix: &Path) -> PathBuf { let prefix = path_to_cstring(prefix); let state_file = unsafe { sys::application_engine_state_file_in_dump(prefix.as_ptr()) }; diff --git a/examples/c-app-engine/src/tests.rs b/examples/c-app-engine/src/tests.rs index c656da7c..194743f3 100644 --- a/examples/c-app-engine/src/tests.rs +++ b/examples/c-app-engine/src/tests.rs @@ -3,7 +3,7 @@ use super::*; -// Only the drain seam is needed here; wallet integration tests cover execution and dumps. +// This fixture covers output borrowing and a zero payload bound; wallet tests cover execution/dumps. // Every non-empty output borrows the same allocation, overwritten by the next drain. #[derive(Default)] struct OutputEngine { @@ -11,6 +11,16 @@ struct OutputEngine { payload: [u8; 4], } +#[unsafe(no_mangle)] +extern "C" fn application_engine_max_method_payload_bytes() -> u64 { + 0 +} + +#[test] +fn empty_method_payload_bound_comes_from_the_engine() { + assert_eq!(EngineApp::max_method_payload_bytes(), 0); +} + #[unsafe(no_mangle)] unsafe extern "C" fn application_engine_drain_output( engine: *mut sys::ApplicationEngine, diff --git a/examples/c-app-sequencer/src/main.rs b/examples/c-app-sequencer/src/main.rs index ce2a970e..cd391b7d 100644 --- a/examples/c-app-sequencer/src/main.rs +++ b/examples/c-app-sequencer/src/main.rs @@ -17,8 +17,7 @@ async fn main() -> ExitCode { fn main() -> ExitCode { eprintln!( "built without an engine, so there is no application to run. Set \ - APPLICATION_ENGINE_LIB, APPLICATION_ENGINE_HEADER and \ - APPLICATION_ENGINE_METHOD_PAYLOAD_LIMIT, then build again." + APPLICATION_ENGINE_LIB and APPLICATION_ENGINE_HEADER, then build again." ); ExitCode::FAILURE } diff --git a/examples/c-wallet-engine/src/lib.rs b/examples/c-wallet-engine/src/lib.rs index 56af2367..05d6b54b 100644 --- a/examples/c-wallet-engine/src/lib.rs +++ b/examples/c-wallet-engine/src/lib.rs @@ -22,14 +22,10 @@ use sequencer_core::application::{ use sequencer_core::l2_tx::{DirectInput, ValidUserOp}; use sequencer_core::user_op::UserOp; -// The header carries no default for the ingress bound, so a build supplies it. This is what -// makes the two agree: a build that told the host a different number than the wallet implements -// fails here rather than at the boundary. -const _: () = assert!( - sys::APPLICATION_ENGINE_MAX_METHOD_PAYLOAD_BYTES as usize - == WalletApp::MAX_METHOD_PAYLOAD_BYTES, - "the payload bound this build declares to the host is not the wallet's own" -); +#[unsafe(no_mangle)] +pub extern "C" fn application_engine_max_method_payload_bytes() -> u64 { + u64::try_from(WalletApp::max_method_payload_bytes()).expect("wallet payload bound exceeds u64") +} thread_local! { /// The last failure's message, and the buffer `state_file_in_dump` answers out of. @@ -367,28 +363,19 @@ fn span_of(payload: &[u8]) -> sys::ApplicationEngineByteSpan { } /// # Safety -/// `engine` is a live handle. +/// `engine` is a live handle and `out_progress` is writable. #[unsafe(no_mangle)] -pub unsafe extern "C" fn application_engine_last_executed_safe_block( +pub unsafe extern "C" fn application_engine_progress( engine: *const ApplicationEngine, -) -> u64 { - unsafe { engine_ref(engine) } - .app - .progress() - .last_executed_safe_block() -} - -/// # Safety -/// `engine` is a live handle. -#[unsafe(no_mangle)] -pub unsafe extern "C" fn application_engine_executed_input_count( - engine: *const ApplicationEngine, -) -> u64 { - unsafe { engine_ref(engine) } - .app - .progress() - .executed_input_count() - .get() + out_progress: *mut sys::ApplicationEngineProgress, +) { + let progress = unsafe { engine_ref(engine) }.app.progress(); + unsafe { + *out_progress = sys::ApplicationEngineProgress { + executed_input_count: progress.executed_input_count().get(), + last_executed_safe_block: progress.last_executed_safe_block(), + } + }; } /// # Safety @@ -407,20 +394,6 @@ pub unsafe extern "C" fn application_engine_create_dump( } } -/// # Safety -/// `prefix` is a NUL terminated path. -#[unsafe(no_mangle)] -pub unsafe extern "C" fn application_engine_delete_dump( - prefix: *const c_char, -) -> sys::ApplicationEngineStatus { - clear_error(); - let prefix = unsafe { path_from(prefix) }; - match WalletApp::delete_dump(&prefix) { - Ok(()) => sys::APPLICATION_ENGINE_STATUS_OK, - Err(err) => report(&err, "delete_dump"), - } -} - /// # Safety /// `prefix` is a NUL terminated path. #[unsafe(no_mangle)] diff --git a/examples/c-wallet-engine/tests/conformance.rs b/examples/c-wallet-engine/tests/conformance.rs index 89d9ccd3..2be1d7cd 100644 --- a/examples/c-wallet-engine/tests/conformance.rs +++ b/examples/c-wallet-engine/tests/conformance.rs @@ -44,6 +44,11 @@ fn deposit(config: WalletConfig, recipient: Address, amount: u64, block: u64) -> #[test] fn abi_mixed_history_matches_native_outputs_progress_and_dump() { let (dir, mut bridge, mut native, config) = fixture(); + assert_eq!( + EngineApp::max_method_payload_bytes(), + WalletApp::max_method_payload_bytes() + ); + assert_eq!(bridge.progress(), native.progress()); let sender = Address::repeat_byte(0x11); let recipient = Address::repeat_byte(0x22); for direct in [ @@ -175,7 +180,7 @@ fn abi_restored_instances_and_checkpoints_are_independent() { std::fs::read(EngineApp::state_file_in_dump(&frozen)).unwrap(), frozen_bytes ); - EngineApp::delete_dump(&source).unwrap(); + std::fs::remove_dir_all(&source).unwrap(); execute_direct_input(&mut second, &input).unwrap(); assert_eq!( state_bytes(&mut second, &dir.path().join("second")), diff --git a/sequencer-core/src/application/mod.rs b/sequencer-core/src/application/mod.rs index 05b2499f..65af049a 100644 --- a/sequencer-core/src/application/mod.rs +++ b/sequencer-core/src/application/mod.rs @@ -152,7 +152,9 @@ impl fmt::Display for InvalidReason { /// Deterministic application state with exclusive ownership and thread transfer. pub trait Application: Send + Sized { - const MAX_METHOD_PAYLOAD_BYTES: usize; + /// Maximum user-op method payload size, stable for this implementation. + /// Zero permits only empty method payloads. + fn max_method_payload_bytes() -> usize; /// Pure validation predicate over current app state: nonce match /// (user replay protection) and fee-balance coverage. Must not @@ -216,7 +218,10 @@ pub trait Application: Send + Sized { /// be a file or directory. A subsequent [`Application::from_dump`] must /// rehydrate equivalent logical state, including progress. Creating the /// dump must preserve the live instance's logical state; later execution - /// of that instance must not change the dump. + /// of that instance must not change the dump. All checkpoint-owned artifacts + /// must reside at or beneath `prefix`; discarding them uses ordinary + /// filesystem deletion and requires no application-specific cleanup. + /// Deletion must leave other checkpoints and restored instances usable. /// /// **Durability**: when this method returns `Ok`, the dump on disk /// must survive an immediate kernel crash. Concretely, the impl @@ -235,9 +240,6 @@ pub trait Application: Send + Sized { /// state file may be the same file when their representations coincide. fn create_dump(&mut self, prefix: &Path) -> Result<(), AppError>; - /// Delete a previously-created dump at `prefix`. - fn delete_dump(prefix: &Path) -> Result<(), AppError>; - /// Path of the canonical state file in a dump at `prefix` (possibly /// `prefix` itself). The returned path must point at a single file. It /// is a pure function of `prefix`: callers may invoke it without @@ -374,7 +376,9 @@ mod tests { } impl Application for ProgressApp { - const MAX_METHOD_PAYLOAD_BYTES: usize = 0; + fn max_method_payload_bytes() -> usize { + 0 + } fn validate_user_op( &self, @@ -423,9 +427,6 @@ mod tests { fn create_dump(&mut self, _prefix: &Path) -> Result<(), AppError> { unreachable!("not used") } - fn delete_dump(_prefix: &Path) -> Result<(), AppError> { - unreachable!("not used") - } fn state_file_in_dump(prefix: &Path) -> PathBuf { prefix.join("state") } diff --git a/sequencer-core/src/scheduler/fold.rs b/sequencer-core/src/scheduler/fold.rs index eff962ed..de340efc 100644 --- a/sequencer-core/src/scheduler/fold.rs +++ b/sequencer-core/src/scheduler/fold.rs @@ -187,7 +187,9 @@ mod tests { } impl Application for FoldApp { - const MAX_METHOD_PAYLOAD_BYTES: usize = 1 + 32 + 20; + fn max_method_payload_bytes() -> usize { + 1 + 32 + 20 + } fn validate_user_op( &self, @@ -229,9 +231,6 @@ mod tests { fn create_dump(&mut self, _prefix: &std::path::Path) -> Result<(), AppError> { unimplemented!("FoldApp does not participate in snapshot lifecycle") } - fn delete_dump(_prefix: &std::path::Path) -> Result<(), AppError> { - unimplemented!("FoldApp does not participate in snapshot lifecycle") - } fn state_file_in_dump(_prefix: &std::path::Path) -> std::path::PathBuf { unimplemented!("FoldApp does not participate in snapshot lifecycle") } diff --git a/sequencer-core/src/scheduler/mod.rs b/sequencer-core/src/scheduler/mod.rs index f19b5288..0caa18bb 100644 --- a/sequencer-core/src/scheduler/mod.rs +++ b/sequencer-core/src/scheduler/mod.rs @@ -481,7 +481,9 @@ mod tests { // Mirrors the wallet app's method-payload cap (selector + amount + // address). A local literal keeps sequencer-core free of an app-core // dependency (which would invert the crate graph). - const MAX_METHOD_PAYLOAD_BYTES: usize = 1 + 32 + 20; + fn max_method_payload_bytes() -> usize { + 1 + 32 + 20 + } fn validate_user_op( &self, @@ -580,10 +582,6 @@ mod tests { unimplemented!("RecordingApp does not participate in snapshot lifecycle") } - fn delete_dump(_prefix: &std::path::Path) -> Result<(), crate::application::AppError> { - unimplemented!("RecordingApp does not participate in snapshot lifecycle") - } - fn state_file_in_dump(_prefix: &std::path::Path) -> std::path::PathBuf { unimplemented!("RecordingApp does not participate in snapshot lifecycle") } diff --git a/sequencer/src/commands/run/startup_hygiene.rs b/sequencer/src/commands/run/startup_hygiene.rs index 0d5256ad..93c47f69 100644 --- a/sequencer/src/commands/run/startup_hygiene.rs +++ b/sequencer/src/commands/run/startup_hygiene.rs @@ -31,19 +31,18 @@ use crate::commands::error::CommandError; use crate::ingress::inclusion_lane::dump_info::{self, delete_dump_dir}; -use sequencer_core::application::Application; /// Run the five-step repair pass (see the module doc for the steps and /// their ordering). -pub(super) fn run_snapshot_hygiene( +pub(super) fn run_snapshot_hygiene( storage: &mut crate::storage::Storage, dumps_dir: &std::path::Path, ) -> Result<(), CommandError> { storage.reset_dump_leases()?; require_finalized_snapshot(storage)?; restamp_finalized_promotion(storage)?; - let gc_removed = snapshot_gc_at_startup::(storage)?; - let sweep_removed = sweep_orphan_dumps::(storage, dumps_dir)?; + let gc_removed = snapshot_gc_at_startup(storage)?; + let sweep_removed = sweep_orphan_dumps(storage, dumps_dir)?; tracing::debug!( gc_removed, sweep_removed, @@ -82,12 +81,10 @@ fn restamp_finalized_promotion(storage: &mut crate::storage::Storage) -> Result< /// finalized, no leases). The companion `sweep_orphan_dumps` then /// catches anything on disk that this leaves behind, plus /// crash-during-create_dump orphans the SQLite layer never saw. -fn snapshot_gc_at_startup( - storage: &mut crate::storage::Storage, -) -> Result { +fn snapshot_gc_at_startup(storage: &mut crate::storage::Storage) -> Result { let removed = storage.gc_unreferenced_dumps()?; for row in &removed { - if let Err(err) = delete_dump_dir::(&row.prefix) { + if let Err(err) = delete_dump_dir(&row.prefix) { tracing::warn!( error = %err, prefix = ?row.prefix, @@ -111,7 +108,7 @@ fn snapshot_gc_at_startup( /// continue (the next startup retries). The post-`require_finalized_snapshot` /// ordering matters: the genesis dump's dir is in /// `list_dump_rows` by the time this runs, so we never delete it. -fn sweep_orphan_dumps( +fn sweep_orphan_dumps( storage: &mut crate::storage::Storage, dumps_dir: &std::path::Path, ) -> Result { @@ -127,7 +124,7 @@ fn sweep_orphan_dumps( if known.contains(&path) { continue; } - match delete_dump_dir::(&path) { + match delete_dump_dir(&path) { Ok(()) => removed += 1, Err(err) => { tracing::warn!( @@ -144,7 +141,7 @@ fn sweep_orphan_dumps( #[cfg(test)] mod tests { use super::*; - use crate::commands::test_support::{SweepTestApp, create_structured_dump}; + use crate::commands::test_support::create_structured_dump; use crate::storage::Storage; use crate::storage::test_helpers::temp_db; @@ -212,7 +209,7 @@ mod tests { create_structured_dump(&orphan_a); std::fs::create_dir(&orphan_b).expect("orphan b dir"); - let removed = sweep_orphan_dumps::(&mut storage, dumps_dir.path()).unwrap(); + let removed = sweep_orphan_dumps(&mut storage, dumps_dir.path()).unwrap(); assert_eq!(removed, 2); assert!(tracked.exists(), "tracked dump must survive"); assert!(!orphan_a.exists()); @@ -225,7 +222,7 @@ mod tests { let mut storage = Storage::open(db.path.as_str()).expect("open"); let dumps_dir = tempfile::tempdir().expect("dumps dir"); - let removed = sweep_orphan_dumps::(&mut storage, dumps_dir.path()).unwrap(); + let removed = sweep_orphan_dumps(&mut storage, dumps_dir.path()).unwrap(); assert_eq!(removed, 0); } @@ -251,7 +248,7 @@ mod tests { // `superseded`'s row is now unreferenced (replaced by // finalized's promotion), but the directory is still on disk. - let removed = snapshot_gc_at_startup::(&mut storage).unwrap(); + let removed = snapshot_gc_at_startup(&mut storage).unwrap(); assert_eq!(removed, 1); assert!(!superseded.exists(), "GC removed the superseded directory"); assert!(finalized.exists(), "current finalized survived"); diff --git a/sequencer/src/commands/run/workers.rs b/sequencer/src/commands/run/workers.rs index c73f9d32..a6d84d18 100644 --- a/sequencer/src/commands/run/workers.rs +++ b/sequencer/src/commands/run/workers.rs @@ -186,7 +186,7 @@ impl PreparedRuntime { // Authority-neutral snapshot repair before the boundary; the five // order-critical steps are documented in `startup_hygiene`. - super::startup_hygiene::run_snapshot_hygiene::(&mut storage, &dumps_dir)?; + super::startup_hygiene::run_snapshot_hygiene(&mut storage, &dumps_dir)?; // Prepare every remaining fallible or awaited dependency before the // authority boundary. Cancellation observes zero workers. @@ -239,7 +239,7 @@ impl PreparedRuntime { let lane_config = InclusionLaneConfig::new(l1_config.identity.batch_submitter_address, dumps_dir) .with_max_batch_open(run_config.max_batch_open()); - let api_config = ApiConfig::new(domain, A::MAX_METHOD_PAYLOAD_BYTES); + let api_config = ApiConfig::new(domain, A::max_method_payload_bytes()); let listener = tokio::net::TcpListener::bind(&run_config.http_addr).await?; let bound_addr = listener.local_addr()?; let snapshot_state = http::SnapshotState { @@ -681,7 +681,9 @@ mod tests { } impl Application for StartupProbeApp { - const MAX_METHOD_PAYLOAD_BYTES: usize = 0; + fn max_method_payload_bytes() -> usize { + 0 + } fn validate_user_op( &self, @@ -733,13 +735,6 @@ mod tests { Ok(()) } - fn delete_dump( - prefix: &std::path::Path, - ) -> Result<(), sequencer_core::application::AppError> { - std::fs::remove_dir_all(prefix)?; - Ok(()) - } - fn state_file_in_dump(prefix: &std::path::Path) -> std::path::PathBuf { prefix.join("state") } diff --git a/sequencer/src/commands/setup/fill.rs b/sequencer/src/commands/setup/fill.rs index e54b2491..5a4882db 100644 --- a/sequencer/src/commands/setup/fill.rs +++ b/sequencer/src/commands/setup/fill.rs @@ -234,7 +234,9 @@ mod tests { } impl Application for CountedSweepTestApp { - const MAX_METHOD_PAYLOAD_BYTES: usize = 0; + fn max_method_payload_bytes() -> usize { + 0 + } fn validate_user_op( &self, @@ -285,10 +287,6 @@ mod tests { Ok(()) } - fn delete_dump(prefix: &Path) -> Result<(), AppError> { - ::delete_dump(prefix) - } - fn state_file_in_dump(prefix: &Path) -> std::path::PathBuf { ::state_file_in_dump(prefix) } diff --git a/sequencer/src/commands/test_support.rs b/sequencer/src/commands/test_support.rs index c4fa3ee4..5ddc0d4b 100644 --- a/sequencer/src/commands/test_support.rs +++ b/sequencer/src/commands/test_support.rs @@ -17,8 +17,7 @@ use sequencer_core::l2_tx::ValidUserOp; use sequencer_core::user_op::UserOp; /// Application stub used in the sweep tests: `create_dump` makes -/// a directory with a marker file inside, `delete_dump` is -/// `remove_dir_all`. The actual marker content is irrelevant — +/// a directory with a marker file inside. The marker content is irrelevant — /// we only care about which directories exist post-sweep. #[derive(Clone, Default)] pub(crate) struct SweepTestApp { @@ -33,7 +32,9 @@ pub(crate) const SweepTestApp: SweepTestApp = SweepTestApp { }; impl Application for SweepTestApp { - const MAX_METHOD_PAYLOAD_BYTES: usize = 0; + fn max_method_payload_bytes() -> usize { + 0 + } fn validate_user_op( &self, _sender: alloy_primitives::Address, @@ -68,10 +69,6 @@ impl Application for SweepTestApp { std::fs::write(prefix.join("state"), b"")?; Ok(()) } - fn delete_dump(prefix: &Path) -> Result<(), AppError> { - std::fs::remove_dir_all(prefix)?; - Ok(()) - } fn state_file_in_dump(prefix: &Path) -> std::path::PathBuf { prefix.join("state") } diff --git a/sequencer/src/http.rs b/sequencer/src/http.rs index eb84b5e7..9e7f5755 100644 --- a/sequencer/src/http.rs +++ b/sequencer/src/http.rs @@ -281,7 +281,7 @@ pub(crate) fn persistent_storage_error(mut error: &(dyn std::error::Error + 'sta pub struct ApiConfig { /// EIP-712 domain user-op signatures are verified against. pub domain: Eip712Domain, - /// The app's `MAX_METHOD_PAYLOAD_BYTES` bound on user-op payloads. + /// The app's `max_method_payload_bytes()` bound on user-op payloads. pub max_user_op_data_bytes: usize, pub max_body_bytes: usize, pub ws_max_subscribers: usize, diff --git a/sequencer/src/ingress/inclusion_lane/dump_info.rs b/sequencer/src/ingress/inclusion_lane/dump_info.rs index cf346004..e143cf9a 100644 --- a/sequencer/src/ingress/inclusion_lane/dump_info.rs +++ b/sequencer/src/ingress/inclusion_lane/dump_info.rs @@ -8,7 +8,7 @@ //! ```text //! dumps// //! state app-owned file or directory — the prefix handed to -//! `Application::{create_dump, from_dump, delete_dump}` +//! `Application::{create_dump, from_dump}` //! info.toml sequencer-owned checkpoint metadata (this module) //! ``` //! @@ -142,17 +142,9 @@ pub fn create_dump_dir_with_info( Ok(()) } -/// Delete one structured dump directory: the app's prefix via its -/// `delete_dump` hook (when present — an orphan from a crash between -/// dir creation and `create_dump` legitimately lacks it), then the -/// rest of the dir (`info.toml` + the dir itself). -pub fn delete_dump_dir(dump_dir: &Path) -> Result<(), AppError> { - let app_prefix = app_prefix(dump_dir); - if app_prefix.exists() { - A::delete_dump(&app_prefix)?; - } - std::fs::remove_dir_all(dump_dir)?; - Ok(()) +/// Delete a checkpoint and its metadata, including incomplete creation remnants. +pub fn delete_dump_dir(dump_dir: &Path) -> io::Result<()> { + std::fs::remove_dir_all(dump_dir) } /// Write `info.toml` into `dump_dir`, durably: temp file, fsync, rename @@ -283,7 +275,9 @@ mod tests { } impl Application for PrefixDumpApp { - const MAX_METHOD_PAYLOAD_BYTES: usize = 0; + fn max_method_payload_bytes() -> usize { + 0 + } fn validate_user_op( &self, @@ -348,15 +342,6 @@ mod tests { Ok(()) } - fn delete_dump(prefix: &Path) -> Result<(), AppError> { - if DIRECTORY { - std::fs::remove_dir_all(prefix)?; - } else { - std::fs::remove_file(prefix)?; - } - Ok(()) - } - fn state_file_in_dump(prefix: &Path) -> PathBuf { if DIRECTORY { prefix.join("progress") @@ -391,8 +376,18 @@ mod tests { assert_eq!(read_info(&dump).unwrap(), sample()); let mut first = PrefixDumpApp::::from_dump(&app_prefix(&dump)).unwrap(); - let second = PrefixDumpApp::::from_dump(&app_prefix(&dump)).unwrap(); - delete_dump_dir::>(&dump).unwrap(); + let mut second = PrefixDumpApp::::from_dump(&app_prefix(&dump)).unwrap(); + let sibling = root.path().join("sibling"); + create_dump_dir_with_info(&mut second, &sibling, &sample()).unwrap(); + delete_dump_dir(&dump).unwrap(); + assert!(!dump.exists(), "the entire checkpoint directory is removed"); + assert_eq!( + PrefixDumpApp::::from_dump(&app_prefix(&sibling)) + .unwrap() + .progress(), + checkpoint, + "other checkpoints survive deletion" + ); execute_direct_input(&mut first, &input).unwrap(); assert_eq!(app.progress(), checkpoint); assert_eq!( @@ -408,7 +403,8 @@ mod tests { first.progress(), "restored state survives source deletion" ); - delete_dump_dir::>(&successor).unwrap(); + delete_dump_dir(&successor).unwrap(); + delete_dump_dir(&sibling).unwrap(); } #[test] diff --git a/sequencer/src/ingress/inclusion_lane/mod.rs b/sequencer/src/ingress/inclusion_lane/mod.rs index 59415dbc..b60bc432 100644 --- a/sequencer/src/ingress/inclusion_lane/mod.rs +++ b/sequencer/src/ingress/inclusion_lane/mod.rs @@ -287,8 +287,7 @@ impl InclusionLane { // Stamp `B` into the freshly finalized dump's info.toml before // GC (the stamp targets the survivor; GC removes the superseded). snapshot::stamp_finalized_promotion(&mut self.storage)?; - let removed = - snapshot::run_gc::(&mut self.storage).map_err(InclusionLaneError::Gc)?; + let removed = snapshot::run_gc(&mut self.storage).map_err(InclusionLaneError::Gc)?; if removed > 0 { tracing::debug!(removed, "post-promotion GC removed unreferenced dumps"); } @@ -528,7 +527,7 @@ fn dequeue_and_execute_user_op_chunk( fn user_op_count_to_bytes(user_op_count: u64) -> u64 { let one_user_op_bytes = SignedUserOp::max_batch_metadata() - .checked_add(A::MAX_METHOD_PAYLOAD_BYTES) + .checked_add(A::max_method_payload_bytes()) .expect("one user-op wire bound overflow: contract-impossible"); let one_user_op_bytes = u64::try_from(one_user_op_bytes).expect("one user-op wire bound must fit in u64"); diff --git a/sequencer/src/ingress/inclusion_lane/snapshot.rs b/sequencer/src/ingress/inclusion_lane/snapshot.rs index c7c0a11c..b3ce78ee 100644 --- a/sequencer/src/ingress/inclusion_lane/snapshot.rs +++ b/sequencer/src/ingress/inclusion_lane/snapshot.rs @@ -70,10 +70,10 @@ pub enum StampError { /// orphan file is acceptable per the no-dangling-row invariant; only /// the reverse (SQLite row pointing at a missing path) would matter, /// and the SQL-first ordering prevents it. -pub(super) fn run_gc(storage: &mut Storage) -> Result { +pub(super) fn run_gc(storage: &mut Storage) -> Result { let removed = storage.gc_unreferenced_dumps()?; for row in &removed { - if let Err(err) = dump_info::delete_dump_dir::(&row.prefix) { + if let Err(err) = dump_info::delete_dump_dir(&row.prefix) { tracing::warn!( error = %err, prefix = ?row.prefix, @@ -314,7 +314,9 @@ mod tests { } impl Application for RecordingDumpApp { - const MAX_METHOD_PAYLOAD_BYTES: usize = 0; + fn max_method_payload_bytes() -> usize { + 0 + } fn validate_user_op( &self, @@ -356,11 +358,6 @@ mod tests { Ok(()) } - fn delete_dump(prefix: &Path) -> Result<(), AppError> { - std::fs::remove_dir_all(prefix)?; - Ok(()) - } - fn state_file_in_dump(prefix: &Path) -> PathBuf { prefix.join("state") } @@ -465,7 +462,7 @@ mod tests { let superseded_prefix = recorded.first().expect("a dump was recorded").clone(); assert!(superseded_prefix.exists(), "pre-GC sanity"); - let removed = super::run_gc::(&mut storage).unwrap(); + let removed = super::run_gc(&mut storage).unwrap(); assert_eq!(removed, 1, "exactly one unreferenced dump cleaned"); assert!(!superseded_prefix.exists(), "filesystem prefix removed too",); @@ -482,7 +479,7 @@ mod tests { take_dump_at_batch_close(&mut app, &mut storage, dumps_dir.path(), 0).unwrap(); // Pending row references the dump; nothing eligible. - let removed = super::run_gc::(&mut storage).unwrap(); + let removed = super::run_gc(&mut storage).unwrap(); assert_eq!(removed, 0); assert!(app.recorded()[0].exists()); } @@ -495,7 +492,9 @@ mod tests { } impl Application for FailingDumpApp { - const MAX_METHOD_PAYLOAD_BYTES: usize = 0; + fn max_method_payload_bytes() -> usize { + 0 + } fn validate_user_op( &self, @@ -536,10 +535,6 @@ mod tests { }) } - fn delete_dump(_prefix: &Path) -> Result<(), AppError> { - Ok(()) - } - fn state_file_in_dump(prefix: &Path) -> PathBuf { prefix.join("state") } diff --git a/sequencer/src/ingress/inclusion_lane/tests.rs b/sequencer/src/ingress/inclusion_lane/tests.rs index 53de8252..f75b5824 100644 --- a/sequencer/src/ingress/inclusion_lane/tests.rs +++ b/sequencer/src/ingress/inclusion_lane/tests.rs @@ -62,7 +62,9 @@ struct TestApp { } impl Application for TestApp { - const MAX_METHOD_PAYLOAD_BYTES: usize = WALLET_MAX_METHOD_PAYLOAD_BYTES; + fn max_method_payload_bytes() -> usize { + WALLET_MAX_METHOD_PAYLOAD_BYTES + } fn validate_user_op( &self, @@ -116,11 +118,6 @@ impl Application for TestApp { Ok(()) } - fn delete_dump(prefix: &Path) -> Result<(), AppError> { - std::fs::remove_dir_all(prefix)?; - Ok(()) - } - fn state_file_in_dump(prefix: &Path) -> PathBuf { prefix.join("state") } @@ -133,7 +130,9 @@ struct InternalUserOpApp { } impl Application for InternalUserOpApp { - const MAX_METHOD_PAYLOAD_BYTES: usize = WALLET_MAX_METHOD_PAYLOAD_BYTES; + fn max_method_payload_bytes() -> usize { + WALLET_MAX_METHOD_PAYLOAD_BYTES + } fn validate_user_op( &self, @@ -177,11 +176,6 @@ impl Application for InternalUserOpApp { Ok(()) } - fn delete_dump(prefix: &Path) -> Result<(), AppError> { - std::fs::remove_dir_all(prefix)?; - Ok(()) - } - fn state_file_in_dump(prefix: &Path) -> PathBuf { prefix.join("state") } @@ -224,7 +218,9 @@ impl SharedCountingApp { } impl Application for SharedCountingApp { - const MAX_METHOD_PAYLOAD_BYTES: usize = WALLET_MAX_METHOD_PAYLOAD_BYTES; + fn max_method_payload_bytes() -> usize { + WALLET_MAX_METHOD_PAYLOAD_BYTES + } fn validate_user_op( &self, @@ -268,11 +264,6 @@ impl Application for SharedCountingApp { Ok(()) } - fn delete_dump(prefix: &Path) -> Result<(), AppError> { - std::fs::remove_dir_all(prefix)?; - Ok(()) - } - fn state_file_in_dump(prefix: &Path) -> PathBuf { prefix.join("state") } @@ -298,7 +289,9 @@ impl Default for ReplayRecordingApp { } impl Application for ReplayRecordingApp { - const MAX_METHOD_PAYLOAD_BYTES: usize = WALLET_MAX_METHOD_PAYLOAD_BYTES; + fn max_method_payload_bytes() -> usize { + WALLET_MAX_METHOD_PAYLOAD_BYTES + } fn validate_user_op( &self, @@ -346,11 +339,6 @@ impl Application for ReplayRecordingApp { Ok(()) } - fn delete_dump(prefix: &Path) -> Result<(), AppError> { - std::fs::remove_dir_all(prefix)?; - Ok(()) - } - fn state_file_in_dump(prefix: &Path) -> PathBuf { prefix.join("state") } @@ -1851,7 +1839,9 @@ impl UserOpCounterApp { } impl Application for UserOpCounterApp { - const MAX_METHOD_PAYLOAD_BYTES: usize = WALLET_MAX_METHOD_PAYLOAD_BYTES; + fn max_method_payload_bytes() -> usize { + WALLET_MAX_METHOD_PAYLOAD_BYTES + } fn validate_user_op( &self, @@ -1894,11 +1884,6 @@ impl Application for UserOpCounterApp { Ok(()) } - fn delete_dump(prefix: &Path) -> Result<(), AppError> { - std::fs::remove_dir_all(prefix)?; - Ok(()) - } - fn state_file_in_dump(prefix: &Path) -> PathBuf { prefix.join("state") } From b18a9a84e357241eda3845347d105f54e23b81b3 Mon Sep 17 00:00:00 2001 From: gcdepaula Date: Sat, 12 Sep 2026 07:07:01 -0300 Subject: [PATCH 07/29] refactor(bindings): promote reusable C integration crates --- .github/workflows/ci.yml | 9 +--- AGENTS.md | 4 +- CLAUDE.md | 4 +- Cargo.toml | 4 +- README.md | 4 +- .../c-app-engine/Cargo.toml | 0 {examples => bindings}/c-app-engine/README.md | 29 +++++++++++- {examples => bindings}/c-app-engine/build.rs | 2 +- .../c-app-engine/include/application-engine.h | 0 .../c-app-engine/src/lib.rs | 0 .../c-app-engine/src/sys.rs | 0 .../c-app-engine/src/tests.rs | 0 .../c-app-sequencer/Cargo.toml | 0 .../c-app-sequencer/build.rs | 0 .../c-app-sequencer/src/lib.rs | 0 .../c-app-sequencer/src/main.rs | 0 docs/protocol/application-contract.md | 2 +- docs/protocol/c-application-binding.md | 10 +++-- examples/c-wallet-engine/Cargo.toml | 2 +- examples/c-wallet-sequencer/Cargo.toml | 2 +- scripts/ci-c-application-smoke.sh | 44 +++++++++++++++++++ 21 files changed, 92 insertions(+), 24 deletions(-) rename {examples => bindings}/c-app-engine/Cargo.toml (100%) rename {examples => bindings}/c-app-engine/README.md (78%) rename {examples => bindings}/c-app-engine/build.rs (98%) rename {examples => bindings}/c-app-engine/include/application-engine.h (100%) rename {examples => bindings}/c-app-engine/src/lib.rs (100%) rename {examples => bindings}/c-app-engine/src/sys.rs (100%) rename {examples => bindings}/c-app-engine/src/tests.rs (100%) rename {examples => bindings}/c-app-sequencer/Cargo.toml (100%) rename {examples => bindings}/c-app-sequencer/build.rs (100%) rename {examples => bindings}/c-app-sequencer/src/lib.rs (100%) rename {examples => bindings}/c-app-sequencer/src/main.rs (100%) create mode 100644 scripts/ci-c-application-smoke.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 803a2a88..5727266e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -63,13 +63,8 @@ jobs: timeout-minutes: 15 run: cargo test --workspace --all-targets --all-features --locked - - name: C application archive path - run: | - cargo build --locked -p c-wallet-engine - APPLICATION_ENGINE_LIB="$PWD/target/debug/libc_wallet_engine.a" \ - APPLICATION_ENGINE_HEADER="$PWD/examples/c-app-engine/include/application-engine.h" \ - cargo build --locked -p c-app-sequencer - ./target/debug/c-app-sequencer --help > /dev/null + - name: C application archive and downstream consumer + run: bash scripts/ci-c-application-smoke.sh canonical-guest: runs-on: ubuntu-latest diff --git a/AGENTS.md b/AGENTS.md index 679cef38..a25bd67d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -147,8 +147,8 @@ Top-level layout follows the system's data flow. Each sequencer module correspon - `sequencer-core/` — shared domain types (`Application`, `SignedUserOp`, `SequencedL2Tx`, `Batch`, `Frame`). - `examples/app-core/` — placeholder wallet app implementing the `Application` trait. - `examples/wallet-sequencer/` — binary crate: wallet app + sequencer library. The model for what an app author builds (their `Application` impl ≙ `app-core`; their binary crate ≙ this). -- `examples/c-app-engine/` — C ABI adapter implementing `Application` for a native engine. -- `examples/c-app-sequencer/` — shared C-engine CLI host and external-archive binary. +- `bindings/c-app-engine/` — reusable C ABI adapter implementing `Application` for a native engine. +- `bindings/c-app-sequencer/` — optional C-engine CLI host and external-archive binary. - `examples/c-wallet-engine/` — reference wallet engine exporting the C ABI, plus its genesis tool and conformance tests. - `examples/c-wallet-sequencer/` — binary composing the C-engine host with the reference wallet engine. - `examples/canonical-app/` — on-chain scheduler reference implementation. diff --git a/CLAUDE.md b/CLAUDE.md index f27ecb8d..037795a4 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -34,8 +34,8 @@ Rust edition 2024 / Axum API / SQLite (rusqlite, WAL) / EIP-712 signing / SSZ en - `sequencer-core/` — shared domain types consumed by both sequencer and scheduler. - `examples/app-core/` — placeholder wallet app implementing `Application`. - `examples/wallet-sequencer/` — binary crate: wallet app + sequencer library. -- `examples/c-app-engine/` — native engine adapter implementing `Application` through a C ABI. -- `examples/c-app-sequencer/` — shared C-engine CLI host and external-archive binary. +- `bindings/c-app-engine/` — reusable native engine adapter implementing `Application` through a C ABI. +- `bindings/c-app-sequencer/` — optional C-engine CLI host and external-archive binary. - `examples/c-wallet-engine/` — reference C ABI exports, genesis tool, and conformance tests. - `examples/c-wallet-sequencer/` — binary composing the C-engine host with the reference wallet engine. - `examples/canonical-app/` — on-chain scheduler reference implementation. diff --git a/Cargo.toml b/Cargo.toml index 67a19a09..4846cc62 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,12 +4,12 @@ members = [ "sequencer", "sequencer-core", "sdk/rust-client", + "bindings/c-app-engine", + "bindings/c-app-sequencer", "examples/app-core", "examples/canonical-app", "examples/canonical-test", "examples/wallet-sequencer", - "examples/c-app-engine", - "examples/c-app-sequencer", "examples/c-wallet-engine", "examples/c-wallet-sequencer", "tests/benchmarks", diff --git a/README.md b/README.md index a0920cfe..01a0ec35 100644 --- a/README.md +++ b/README.md @@ -260,8 +260,8 @@ released even on client disconnect. - `sequencer/src/storage/`: schema, migrations, SQLite persistence (split per writer role), and replay reads - `sequencer-core/src/`: shared domain types and interfaces (`Application`, `SignedUserOp`, `SequencedL2Tx`, feed message types) - `examples/app-core/src/`: wallet prototype implementing `Application` -- [`examples/c-app-engine/`](examples/c-app-engine/README.md): C ABI adapter and external static-archive integration guide -- `examples/c-app-sequencer/`: shared C-engine CLI host and external-archive binary +- [`bindings/c-app-engine/`](bindings/c-app-engine/README.md): reusable C ABI adapter and external static-archive integration guide +- `bindings/c-app-sequencer/`: optional C-engine CLI host and external-archive binary - `examples/c-wallet-engine/`: reference wallet C ABI exports, genesis tool, and conformance tests - `examples/c-wallet-sequencer/`: binary composing the C-engine host with the reference wallet engine - `tests/benchmarks/`: benchmark harnesses and benchmark spec diff --git a/examples/c-app-engine/Cargo.toml b/bindings/c-app-engine/Cargo.toml similarity index 100% rename from examples/c-app-engine/Cargo.toml rename to bindings/c-app-engine/Cargo.toml diff --git a/examples/c-app-engine/README.md b/bindings/c-app-engine/README.md similarity index 78% rename from examples/c-app-engine/README.md rename to bindings/c-app-engine/README.md index 254ec262..d8ffb17c 100644 --- a/examples/c-app-engine/README.md +++ b/bindings/c-app-engine/README.md @@ -1,5 +1,8 @@ # C application bridge +The crates under `bindings/` are reusable integration libraries. The wallet +engine and binary under `examples/` demonstrate their use. + `EngineApp` implements `sequencer_core::application::Application` using the [application-engine C ABI](include/application-engine.h). The sequencer owns one engine at a time. A handle can move between threads; calls on it never overlap. @@ -59,7 +62,9 @@ panic follows the shared terminal-error policy. ## External engine -Build the application's static archive and use the corresponding header: +Build the application's static archive against the header from the chosen +sequencer revision. From a checkout of that revision, build the generic host +with that same header: ```sh APPLICATION_ENGINE_LIB=/absolute/path/libengine.a \ @@ -75,6 +80,28 @@ cross this ABI. With no external archive configured, the generic binary reports that no engine was linked, and `c-wallet-sequencer` supplies the reference implementation through Cargo. +For a binary in another repository, depend on the host directly. Replace +`` with the sequencer revision whose header the engine uses: + +```toml +[dependencies] +c-app-sequencer = { git = "https://github.com/cartesi/sequencer", rev = "" } +tokio = { version = "1", features = ["macros", "rt-multi-thread"] } +``` + +```rust +#[tokio::main] +async fn main() -> std::process::ExitCode { + c_app_sequencer::run().await +} +``` + +Set the same `APPLICATION_ENGINE_LIB` and `APPLICATION_ENGINE_HEADER` variables +when running `cargo build` in that binary's repository. The host's `run()` owns +CLI parsing and tracing setup. For custom host wiring, depend on `c-app-engine` +and `sequencer` at the same Git revision and compose `EngineApp` with +`sequencer::run_command` directly. No wallet crate is required in either case. + The conformance suite compares native and ABI execution over mixed inputs, notices and vouchers, rejection/no-op progress, dump round trips, independent instances, and fatal/error classification. diff --git a/examples/c-app-engine/build.rs b/bindings/c-app-engine/build.rs similarity index 98% rename from examples/c-app-engine/build.rs rename to bindings/c-app-engine/build.rs index 1307dcbf..34c43379 100644 --- a/examples/c-app-engine/build.rs +++ b/bindings/c-app-engine/build.rs @@ -108,7 +108,7 @@ fn main() { let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR")); let header = match env::var("APPLICATION_ENGINE_LIB") { Ok(engine_lib) => external_engine(&engine_lib), - // Linked from inside this workspace, where `c-wallet-engine` is the engine + // The final binary supplies the engine when no external archive is configured. Err(_) => { assert!( env::var_os("APPLICATION_ENGINE_HEADER").is_none(), diff --git a/examples/c-app-engine/include/application-engine.h b/bindings/c-app-engine/include/application-engine.h similarity index 100% rename from examples/c-app-engine/include/application-engine.h rename to bindings/c-app-engine/include/application-engine.h diff --git a/examples/c-app-engine/src/lib.rs b/bindings/c-app-engine/src/lib.rs similarity index 100% rename from examples/c-app-engine/src/lib.rs rename to bindings/c-app-engine/src/lib.rs diff --git a/examples/c-app-engine/src/sys.rs b/bindings/c-app-engine/src/sys.rs similarity index 100% rename from examples/c-app-engine/src/sys.rs rename to bindings/c-app-engine/src/sys.rs diff --git a/examples/c-app-engine/src/tests.rs b/bindings/c-app-engine/src/tests.rs similarity index 100% rename from examples/c-app-engine/src/tests.rs rename to bindings/c-app-engine/src/tests.rs diff --git a/examples/c-app-sequencer/Cargo.toml b/bindings/c-app-sequencer/Cargo.toml similarity index 100% rename from examples/c-app-sequencer/Cargo.toml rename to bindings/c-app-sequencer/Cargo.toml diff --git a/examples/c-app-sequencer/build.rs b/bindings/c-app-sequencer/build.rs similarity index 100% rename from examples/c-app-sequencer/build.rs rename to bindings/c-app-sequencer/build.rs diff --git a/examples/c-app-sequencer/src/lib.rs b/bindings/c-app-sequencer/src/lib.rs similarity index 100% rename from examples/c-app-sequencer/src/lib.rs rename to bindings/c-app-sequencer/src/lib.rs diff --git a/examples/c-app-sequencer/src/main.rs b/bindings/c-app-sequencer/src/main.rs similarity index 100% rename from examples/c-app-sequencer/src/main.rs rename to bindings/c-app-sequencer/src/main.rs diff --git a/docs/protocol/application-contract.md b/docs/protocol/application-contract.md index 8d312ba6..8acd89df 100644 --- a/docs/protocol/application-contract.md +++ b/docs/protocol/application-contract.md @@ -12,7 +12,7 @@ This document **owns** the contract. [`AGENTS.md`](../../AGENTS.md) is the map. The [wallet](../../examples/app-core/) is the reference implementation. A production application may execute natively or wrap a Cartesi Machine. The [C application binding](c-application-binding.md) adapts this contract to -native engines; its [build guide](../../examples/c-app-engine/README.md) includes +native engines; its [build guide](../../bindings/c-app-engine/README.md) includes a reference wallet integration. ## The execution methods diff --git a/docs/protocol/c-application-binding.md b/docs/protocol/c-application-binding.md index 739be090..ef3f6981 100644 --- a/docs/protocol/c-application-binding.md +++ b/docs/protocol/c-application-binding.md @@ -1,10 +1,12 @@ # C application binding An application exposes the -[`application-engine.h`](../../examples/c-app-engine/include/application-engine.h) +[`application-engine.h`](../../bindings/c-app-engine/include/application-engine.h) C ABI in a static archive. `c-app-engine::EngineApp` adapts that engine to the -Rust `Application` trait. The [build guide](../../examples/c-app-engine/README.md) -covers linking, genesis, host commands, and reference tests. +Rust `Application` trait. These reusable bindings live under `bindings/`; +`c-app-sequencer` supplies an optional CLI host. The +[build guide](../../bindings/c-app-engine/README.md) covers downstream dependencies, +linking, genesis, host commands, and the reference wallet under `examples/`. ## Contract ownership @@ -13,7 +15,7 @@ covers linking, genesis, host commands, and reference tests. - [Scheduler semantics](scheduler-semantics.md) defines canonical ordering and the acceptance boundary. The native engine does not acquire an ordering role through this ABI. -- The [C header](../../examples/c-app-engine/include/application-engine.h) +- The [C header](../../bindings/c-app-engine/include/application-engine.h) defines record layout, statuses, pointer ownership, and call lifetimes. An engine archive and its generated Rust bindings must agree on that header. - [Snapshot lifecycle](../snapshots/lifecycle.md) owns checkpoint registration, diff --git a/examples/c-wallet-engine/Cargo.toml b/examples/c-wallet-engine/Cargo.toml index 6099778c..82cfdd9c 100644 --- a/examples/c-wallet-engine/Cargo.toml +++ b/examples/c-wallet-engine/Cargo.toml @@ -21,7 +21,7 @@ path = "src/bin/c-wallet-genesis.rs" [dependencies] app-core = { path = "../app-core" } -c-app-engine = { path = "../c-app-engine" } +c-app-engine = { path = "../../bindings/c-app-engine" } sequencer-core = { path = "../../sequencer-core" } alloy-primitives = { workspace = true } diff --git a/examples/c-wallet-sequencer/Cargo.toml b/examples/c-wallet-sequencer/Cargo.toml index 06080aa3..31402723 100644 --- a/examples/c-wallet-sequencer/Cargo.toml +++ b/examples/c-wallet-sequencer/Cargo.toml @@ -10,6 +10,6 @@ readme = "../../README.md" authors.workspace = true [dependencies] -c-app-sequencer = { path = "../c-app-sequencer" } +c-app-sequencer = { path = "../../bindings/c-app-sequencer" } c-wallet-engine = { path = "../c-wallet-engine" } tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } diff --git a/scripts/ci-c-application-smoke.sh b/scripts/ci-c-application-smoke.sh new file mode 100644 index 00000000..eb9cf34d --- /dev/null +++ b/scripts/ci-c-application-smoke.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# Link an external engine into the generic host and an independent Cargo consumer. +set -euo pipefail + +root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "${root}" +export CARGO_TARGET_DIR="${root}/target" + +unset APPLICATION_ENGINE_LIB APPLICATION_ENGINE_HEADER +cargo build --locked -p c-wallet-engine +export APPLICATION_ENGINE_LIB="${CARGO_TARGET_DIR}/debug/libc_wallet_engine.a" +export APPLICATION_ENGINE_HEADER="${root}/bindings/c-app-engine/include/application-engine.h" +cargo build --locked -p c-app-sequencer +"${CARGO_TARGET_DIR}/debug/c-app-sequencer" --help >/dev/null + +smoke_dir="$(mktemp -d "${TMPDIR:-/tmp}/sequencer-c-consumer.XXXXXX")" +trap 'rm -rf "${smoke_dir}"' EXIT +ln -s "${root}" "${smoke_dir}/source" +consumer_dir="${smoke_dir}/consumer" +mkdir -p "${consumer_dir}/src" +cat >"${consumer_dir}/Cargo.toml" <<'TOML' +[package] +name = "c-application-consumer-smoke" +version = "0.0.0" +edition = "2024" + +[workspace] + +[dependencies] +c-app-sequencer = { path = "../source/bindings/c-app-sequencer" } +tokio = { version = "1", features = ["macros", "rt-multi-thread"] } +TOML +cat >"${consumer_dir}/src/main.rs" <<'RUST' +#[tokio::main] +async fn main() -> std::process::ExitCode { + c_app_sequencer::run().await +} +RUST + +# Reuse CI's pinned dependencies, allowing Cargo to adjust only the consumer's lockfile. +cp Cargo.lock "${consumer_dir}/Cargo.lock" +cargo build --manifest-path "${consumer_dir}/Cargo.toml" --offline +"${CARGO_TARGET_DIR}/debug/c-application-consumer-smoke" --help >/dev/null +echo "C application archive and downstream consumer smoke passed" From 71b3b35da22b3226f22ccca745ebe86f07f61e65 Mon Sep 17 00:00:00 2001 From: gcdepaula Date: Wed, 16 Sep 2026 04:38:08 -0300 Subject: [PATCH 08/29] feat: add versioned canonical history read foundation Validate history claims before inclusive canonical pagination and capture snapshot history identity with the artifact lease. Preserve existing HTTP and WebSocket behavior for the coordinated consumer cutover. Cover recovery replacement, coherent reads, rebuilt history bases, deep backlogs, and lease metadata. Record the accepted snapshot-to-tip workflow and remaining API work. --- docs/invariants.md | 3 +- docs/plans/2026-07-coordination-tracks.md | 55 +- .../2026-07-track3-feed-replay-design.md | 514 +++++++++--------- docs/review/register.md | 4 +- docs/snapshots/lifecycle.md | 5 + sequencer-core/src/history.rs | 165 ++++++ sequencer/src/egress/l2_tx_feed/mod.rs | 13 +- sequencer/src/storage/egress.rs | 149 +++-- sequencer/src/storage/egress/canonical.rs | 157 ++++++ .../src/storage/egress/canonical/tests.rs | 432 +++++++++++++++ sequencer/src/storage/mod.rs | 2 +- sequencer/src/storage/snapshot_dumps.rs | 186 ++++++- 12 files changed, 1273 insertions(+), 412 deletions(-) create mode 100644 sequencer/src/storage/egress/canonical.rs create mode 100644 sequencer/src/storage/egress/canonical/tests.rs diff --git a/docs/invariants.md b/docs/invariants.md index e319fd1f..cf56da55 100644 --- a/docs/invariants.md +++ b/docs/invariants.md @@ -591,7 +591,8 @@ like a simplification and would break a registered invariant: `storage/mutations.rs`); `executed_inputs` constraints and invalidation trigger (`storage/migrations/0001_schema.sql`); storage-derived `H` (`storage/history.rs`); snapshot count checks - (`storage/snapshot_dumps.rs`); and pre-execution catch-up checks + (`storage/snapshot_dumps.rs`); coherent egress bounds/pages with contiguous + count checks (`storage/egress/canonical.rs`); and pre-execution catch-up checks (`ingress/inclusion_lane/catch_up.rs`). - **Depended on by:** restart determinism, standard-recovery rollback/reuse, post-cockroach continuation at `K`, snapshot coherence, and the future diff --git a/docs/plans/2026-07-coordination-tracks.md b/docs/plans/2026-07-coordination-tracks.md index c5cc56a4..49459d97 100644 --- a/docs/plans/2026-07-coordination-tracks.md +++ b/docs/plans/2026-07-coordination-tracks.md @@ -14,7 +14,7 @@ freely at this stage — no backward-compatibility constraints. |---|-------|-------|--------| | 1 | WS context fields + L1 provenance (PR #26) | Stephen | **done** — merged to main | | 2 | Restore `docs/review/` ledger + this plan | us | **done** | -| 3 | Feed & replay protocol redesign | us (design) → us/Stephen (impl) | **storage foundation landed; public API open** — the [Track 3 ordered handoff](2026-07-track3-feed-replay-design.md#7-ordered-implementation-handoff) exclusively owns its sequence and decision gates | +| 3 | Feed & replay protocol redesign | us (design) → us/Stephen (impl) | **internal read foundation implemented; consumer API open** — the [Track 3 ordered handoff](2026-07-track3-feed-replay-design.md#7-ordered-implementation-handoff) owns the accepted workflow and remaining cutover | | 4 | Storage decode policy | us | **done** — fail-loud for contract-impossible values; the named `saturating_query_bound` only where clamping preserves the predicate (policy lives in `storage/convert.rs` + the invariants check policy) | | 5 | Fee exponentiation LUT | us | **deferred** — decided exact-floor if built (the table *is* the spec, algorithm-free; replay continuity across the upgrade explicitly not preserved); a separate pending design decision may make log-space fees defunct — revisit after syncing with Bart | | 6 | Dump / `Application` API redesign | us + Bart | **revised interface implemented** — [Application contract](../protocol/application-contract.md); native bridge conformance is a separate integration branch | @@ -23,9 +23,8 @@ freely at this stage — no backward-compatibility constraints. **Current campaign order:** -1. Land the authority-boundary + durable-history-foundation branch (squashed, - review complete — ready for its PR against main). -2. Implement Track 3's public protocol on a focused successor branch. +1. Review the Track 3 internal history-read and snapshot-metadata foundation. +2. Implement the coordinated Track 3 HTTP/WS/SDK cutover on its successor branch. 3. Validate Track 6 against the reference C bridge, then the private DEX engine when shared. 4. Track 5 (fee LUT) only after the log-space-fees decision. @@ -34,37 +33,23 @@ Deferred (revisit with libdex rollout): multi-file/tar snapshot serving ## Track 3 — Feed & replay protocol redesign -The current protocol grew ad hoc; the redesign is type-first and covers the -whole consumer data-access story: paginated finalized-history endpoints plus -the live subscription, composable without races. The -[design doc](2026-07-track3-feed-replay-design.md) owns the requirements and -the ordered implementation handoff; the storage/recovery foundation -(era/generation metadata, canonical `ExecutedInputCount` attribution, -snapshot/catch-up verification) is landed, while `GET /history-version`, -replay routes, gold-boundary projection, and WS v2 remain open. - -Settled decisions the implementation must respect: - -- **Feed coordinate:** `Application::executed_input_count()`, not SQLite - rowid. An application at count `X` subscribes at `X`, consumes entry `X`, - advances to `X + 1`. Standard recovery may reuse suffix offsets under a new - generation; cockroach recovery records the folded count `K` as the era's - available-history base, and requests below `K` fail with `available_from` - plus the bootstrap recipe. -- **Discontinuity detection is pull-based.** A crash or danger-detector exit - cannot send a farewell frame, so the load-bearing contract is the required - subscription claim `{era_id, recovery_generation, offset}` plus a - current-pair endpoint; in-band disconnect errors are best-effort only. - Bart confirmed the scalar generation contract (2026-07-28); the `EraId` - generalization and changed-era bootstrap behavior still need his consumer - review and are not attributed to that confirmation. -- **Event framing:** per-row denormalized context (as shipped in PR #26); - no `FrameSealed`/`BatchSealed` boundary events unless a consumer - demonstrates the row context cannot express its need. -- **Clock:** application time is safe-block based. Direct inputs execute at - their exact inclusion block; user ops at their frame's safe block. - `block_timestamp` may ride as provenance but is never an application - transition input (see the application contract). +Infrastructure subscribers download an application-defined snapshot over HTTP, +restore their application, and use one WS stream for both canonical backlog and +live inputs. The [design](2026-07-track3-feed-replay-design.md) owns the history +claims, typed refusals, resource bounds, and fresh-snapshot recovery workflow. +Raw `/inputs` and separate HTTP transaction replay are outside this feature. +The watchdog retains its independent trusted-state/L1 comparison workflow. + +The internal foundation provides typed history claims and policy errors, +coherent history-bound reads, inclusive canonical pagination, and history +identity captured with a snapshot's lease and count. Existing HTTP/WS responses +still expose physical cursors. The consumer cutover updates snapshot metadata, +WS admission/replay, SDK, and harness together; it must also remove the total +catch-up cap while retaining bounded pages, queues, and subscriber counts. + +Close the consumer invalidation finding only after that cutover and its +recovery/bootstrap acceptance tests. The next PR must demonstrate cold start, +ordinary resume, fresh-snapshot recovery, and gap-free backlog-to-tip delivery. ## Track 5 — Fee exponentiation LUT (deferred) diff --git a/docs/plans/2026-07-track3-feed-replay-design.md b/docs/plans/2026-07-track3-feed-replay-design.md index 78b71470..95c5a328 100644 --- a/docs/plans/2026-07-track3-feed-replay-design.md +++ b/docs/plans/2026-07-track3-feed-replay-design.md @@ -1,40 +1,43 @@ # Feed & Replay Protocol Design (Track 3) -**Status: accepted architecture; storage/execution foundation landed; public -protocol open.** The team accepted the architecture recorded here; Bart's -consumer review remains open only for the decision gates named in §8. The -history identity `(EraId, RecoveryGeneration)` was accepted 2026-08-01; the -authoritative `Application::executed_input_count` offset and cockroach-base -semantics were accepted 2026-08-02. Durable DB metadata, the typed -application-execution boundary, per-input canonical attribution, and -snapshot/catch-up agreement are -landed. `GET /history-version`, replay routes, and the WS projection remain -open. The consumer bootstrap and `/inputs` questions remain decision gates. -The completed protocol will supersede the ad-hoc WS protocol and close the -open WS invalidation-contract finding (see the review register) only -when the ordered handoff in §7 is complete. At that point, the -normative parts graduate into `docs/protocol/` and the README is rewritten. - -## 1. Motivation - -The current protocol is an unframed infinite stream of a two-variant enum over -a WS socket, with every session-level signal smuggled into transport close -frames. Three defects drive the redesign: - -- **The open invalidation-contract finding (register):** no - invalidation/rollback signal. Today, recovery - cascades hide already-streamed rowid-addressed rows and re-sequence - replacements at higher rowids under a reused batch nonce. A cursor-resumed - mirror can silently diverge. -- **No historical bootstrap:** the catch-up window is shallow; a consumer - cannot build state from genesis over the feed. -- **No control plane:** catch-up-to-live is unmarked and continuity errors are - prose close reasons rather than typed messages. - -Consumer model (Bart's libdex is the concrete instance): replay finalized -history via HTTP, then subscribe for the soft tip while maintaining a bit-exact -mirror of application state. Recovery and rebuild boundaries must be detectable -and have explicit remediation. +**Status: internal read foundation implemented; consumer API cutover pending.** +Typed history claims and refusals, coherent canonical pages, and history +identity captured with snapshot leases are implemented alongside the durable +history/execution foundation. HTTP snapshot metadata, the canonical-coordinate +WS protocol, and the matching SDK/consumer workflow remain to be implemented. The current API contract is +in the [README](../../README.md); the [ordered handoff](#7-ordered-implementation-handoff) +defines the remaining work. Close the WS invalidation-contract finding only +after the consumer cutover and its acceptance tests. + +## 1. Consumer workflow and scope + +A subscriber starts cold by downloading an application-defined snapshot over +HTTP, restores its replicated application, and subscribes from that state's +executed-input count. One WS stream supplies every subsequent committed input +in order, then continues following the tip. Recovery and rebuild boundaries +must be detected before the consumer applies inputs from a different history. +The application owns the restore artifact and its format; egress transports +that artifact with enough metadata to identify the state it contains. + +The watchdog has a different trust boundary: it starts from trusted state, +advances independently using L1 inputs, and fetches the matching finalized +comparison artifact. Its finalized-state HTTP endpoints remain available. +The egress API serves both consumers exclusively within the operator's own +infrastructure, with network access controls. + +HTTP fits a finite snapshot download and already has file streaming and leases. +WS fits the existing ordered-input feed: stored and newly committed inputs use +the same replay loop. Splitting finalized replay onto HTTP would add another +input-fetching path and a moving-boundary handoff without helping this replica +workflow. Raw `/inputs` and paginated HTTP `/l2-txs` are outside this scope; +revisit them for a concrete archival or historical-query consumer. WS carries +backlog on both sides of the gold boundary. + +The current snapshot-then-subscribe path already supports ordinary bootstrap, +but its physical rowid cursor cannot detect recovery discontinuities. Its total +catch-up limit can also reject a valid snapshot whose download/restore or +preceding execution leaves too much backlog. The new protocol addresses those +boundaries without requiring different checkpoint timing. ## 2. Concepts and coordinates @@ -43,18 +46,23 @@ and have explicit remediation. - **Feed coordinate `offset: ExecutedInputCount`** — the authoritative `Application::executed_input_count()` boundary, starting at zero. An application at `X` is ready to consume history entry `X`; applying that - entry advances it to `X + 1`. SQLite now stores this as a sparse canonical + entry advances it to `X + 1`. SQLite stores this as a sparse canonical attribution beside its append-only physical rowid replay log. The current - public feed still exposes rowid and changes only at the later API cutover. + feed still exposes rowid and changes only at the API cutover. - **Era base `K`** — the smallest feed offset locally available in this era. Genesis setup starts at zero. Cockroach recovery sets it to `S'.executed_input_count()` after the fold; absolute offsets continue, but the unavailable prefix is not reconstructed. `K` is application history, not the snapshot's physical `l2_tx_index`: recovery cursor-padding rows may advance the latter without executing an application input. +- **Snapshot count `C`** — the exact executed-input count in a selected + application dump. The replica restores at `C` and requests input `C` next. + A current snapshot may be newer than `K`; retaining a snapshot exactly at + `K` is not required. - **Gold boundary `G`** — within one era, the exclusive executed-input count after the scheduler-accepted prefix. Entries with `offset < G` cannot be - invalidated; `G` only advances. + invalidated; `G` only advances. It does not restrict WS admission or + determine which transport carries an input. - **Era ID `e`** — random durable UUIDv4 minted write-once in one era's baseline transaction. Cockroach recovery/fresh setup creates a new era because the rebuilt DB cannot serve the prior era's ordered L2 history from @@ -90,20 +98,27 @@ and have explicit remediation. unsupported. Operating copied state as a new era requires explicit fresh/wiped-directory setup/rebuild; detecting uncoordinated clones requires external authority. -- The protocol will expose the pair through `GET /history-version`; every - subscribe request will claim it. WS responses repeat only the recovery - generation, including best-effort in-band `discontinuity` events. None of - those API changes is part of the landed storage foundation. - -Within the same era, a stale generation means “discard and replay the soft -suffix.” Standard recovery restores the retained application state, so its -count rolls back, then advances it over replacement force-drained directs; the -same suffix offsets may therefore name different inputs under the new -generation. A changed era means the client reacquires a current-era -snapshot/bootstrap even if its old state's numeric count happens to be in the -new era's range. - -Cockroach recovery now leaves the rebuild base NULL at baseline creation, then +- The snapshot response carries the pair together with the dump's count; + every subscribe request claims that pair. WS responses carry the admitted + recovery generation. The bootstrap workflow requires no separate + `/history-version` request. Reading current metadata cannot authorize an old + state: the client keeps the history identity associated with its own state. + +Standard recovery restores the retained application state, so its count rolls +back, then advances over replacement force-drained directs. The same suffix +count may name a different input under a new generation. The initial client +workflow handles both a stale generation and a changed era by discarding its +replica and downloading a current snapshot. It does this even when the old +state's numeric count is in the new history's range. Retaining a known-stable +checkpoint for cheaper rollback is a possible later client optimization. + +Recovery and rebuild run before runtime admission across a process boundary. +Existing subscriptions end before history changes. Each admitted session is +bound to its validated history version; reconnect validation is the correctness +boundary. A guaranteed farewell, invalidation broadcaster, or generation-polling +worker is unnecessary under this lifecycle. + +Cockroach recovery leaves the rebuild base NULL at baseline creation, then binds `K = S'.executed_input_count()` in the same transaction that registers the initial finalized snapshot. Setup completion refuses until both exist. Requests below `K` receive a typed `history_unavailable` response carrying @@ -134,239 +149,196 @@ The physical and logical coordinates deliberately remain separate: There is no backfill, repair, or neighbor-derived fallback. A missing, extra, or wrong attribution is a terminal self-invariant failure. This keeps the -public API cutover a projection over already-correct durable values rather than +API cutover a projection over already-correct durable values rather than the moment those values first become authoritative. -## 4. Historical replay endpoints (HTTP, paginated, finalized-only) +## 4. HTTP snapshot bootstrap -Both endpoints serve immutable entries and therefore carry no recovery -generation. Every response carries `era_id`; an entry already returned within -that era never changes. Tail pages and their current end/gold metadata can -advance, so clients and caches must revalidate the current tail rather than -treating every page response as immutable. A client must never splice pages -from different eras. +`GET /latest_snapshot` supplies the latest available application snapshot: +latest valid pending snapshot if present, otherwise finalized. The response +carries the selected dump's canonical count `C` and history version `(e, g)`. +These are selected together with the artifact's lease in one coherent storage +transaction. They describe that artifact at acquisition, not a separately read +live head after the download. -### 4.1 `GET /inputs?from_index=N&limit=K` - -Raw InputBox order over `safe_inputs`: direct inputs and our own batches, -exactly as L1 ordered them, with PR #26 provenance: +Logical response metadata, with header spellings fixed during the API cutover: ```json { "era_id": "550e8400-e29b-41d4-a716-446655440000", - "items": [ - { - "input_index": 7, - "sender": "0x...", - "payload": "0x...", - "block_number": 123, - "block_timestamp": 1700000000, - "transaction_hash": "0x..." - } - ], - "next_index": 8, - "end_index": 41 + "recovery_generation": 7, + "executed_input_count": 100 } ``` -`end_index` is the current exclusive upper bound. Rows appear once the L1 safe -head passes them. - -### 4.2 `GET /l2-txs?from_offset=N&limit=K` - -Feed order capped at the gold boundary. Items use the same shapes as WS data -events so replay pages and live events are interchangeable mirror inputs: - -```json -{ - "era_id": "550e8400-e29b-41d4-a716-446655440000", - "items": [], - "next_offset": 101, - "gold_boundary": 250 -} -``` - -`from_offset = N` is inclusive: the first item, if present, is history entry -`N`; `next_offset` is the application count after all returned entries. The -server never serves `offset >= G` here. A dedicated indexed query computes `G` -per page. Because it only advances, a stale read is conservative. Requests -below the era base fail with `history_unavailable` rather than pretending the -missing prefix is empty. +The response body is the application-defined restore artifact. The current +handler opens the single file named by `Application::state_file_in_dump`. +Integration must demonstrate that the delivered artifact restores the intended +replica, including progress; the application contract permits recovery dumps +whose full representation differs from that comparison file. This is an +application/adapter integration obligation, not an egress-owned state format. + +The lease protects the artifact through response completion or disconnect. +Once downloaded, the consumer owns its copy and does not depend on the server +retaining that dump. A snapshot exactly at `K` need not remain available: a +retained snapshot at `C >= K` initializes the replica, which replays from `C`. +Recovery during download or restoration can invalidate the claim; subscription +validation handles that race without holding intake or history advancement. + +The client verifies the restored application's count against `C`. Cache +validators must distinguish the era and selected artifact, including a rebuilt +era at the same inclusion block. Cached bytes must retain their matching +history metadata; a fresh version lookup must not relabel cached old state. +Any conditional response must preserve that association. Range resumption is a +possible later transport feature, not a prerequisite for cold bootstrap. + +`GET /finalized_state` and its metadata route keep serving the watchdog's +comparison workflow. Their artifact metadata and cache identity must likewise +refer to the selected finalized checkpoint. The watchdog's trusted starting +state and independent L1 replay are not replaced by the tip-replica bootstrap. ## 5. WS subscription v2 -### 5.1 Handshake and continuity - -`GET /ws/subscribe?from_offset=N&era_id=e&recovery_generation=g` upgrades. All -three coordinates are required: the client claims both the history reality it -holds and the exact application boundary it is ready to execute. - -```json -{ - "kind": "hello", - "recovery_generation": 4, - "available_from": 100, - "live_head": 312, - "gold_boundary": 250, - "max_subscribers": 64 -} -``` - -- Every server WS message carries the current `recovery_generation`; no WS - response repeats `EraId`. The era is an admission credential, and a client - obtains the current value through the bootstrap/history-version path. -- If `era_id` differs, send `era_changed`, then close 1008. Old application - state cannot be resumed merely because its count is numerically in range; - the client reacquires the current-era snapshot/bootstrap. -- If the era matches but `recovery_generation` differs, send - `stale_generation` with the current generation, then close 1008. The client - rebuilds only its soft suffix above its last known `G`. -- If `from_offset < K`, send `history_unavailable { available_from: K, ... }`, - then close 1008. The client acquires the snapshot/bootstrap at `K`. - -### 5.2 Depth guarantee - -Within one era, every `from_offset` in `[G, H]` is serveable: WS supplies the -soft-history entries in `[G, H)`, while a request exactly at `H` joins live and -waits for the next input. Section 8 owns the policy for offsets above `H`. HTTP -replay covers entries below `G`. A request in `[K, G)` receives: - -```json -{ - "kind": "error", - "recovery_generation": 4, - "error": "below_gold_boundary", - "gold_boundary": 250 -} -``` - -then a 1008 close. The client loop is: - -1. Page `/l2-txs` until history is exhausted at `G0`, verifying one `era_id` - across every page. -2. Subscribe from `G0` with the pair from `/history-version` (or the - current-era bootstrap response). -3. On `stale_generation`, discard the soft suffix and return to step 1. On - `below_gold_boundary`, return to step 1. On `era_changed`, reacquire the - current-era snapshot/bootstrap and restart. - -There is no same-era continuity hole if `G` advances between replay and -subscribe: entries in `[G0, G)` remain finalized and serveable over HTTP, and -the typed `below_gold_boundary` response sends the client back through that -replay. Both HTTP and WS use the same boundary convention: an application at -count `X` requests `X`, and the first returned input is entry `X`. - -### 5.3 Events - -Data events retain PR #26's denormalized row context (`user_op` / -`direct_input` with nonce, safe block, batch nonce, input index, block -timestamp, and transaction hash). Bart confirmed this field set suffices for a -bit-exact mirror. Normalized frame/batch boundary events remain rejected until -a consumer demonstrates a need. - -Every event carries `recovery_generation`. Control events join the same tagged -enum: - -- `hello` — §5.1; -- `live` — sent once catch-up reaches the live head; -- `discontinuity { recovery_generation }` — best-effort only; reconnect - validation is load-bearing; -- `error { error, ... }` — typed policy error followed by close 1008. - -### 5.4 Wire format - -JSON text frames, serde-tagged by `kind`, with mandatory new fields. We break -old clients freely. `WsTxMessage = BroadcastTxMessage` remains the shared -exhaustive SDK/server type. - -## 6. What this replaces - -- `WS_CATCHUP_WINDOW_EXCEEDED_REASON` and the `live_start_offset` prose close - reason — replaced by `below_gold_boundary` plus `/l2-txs`. -- The interim “rebuild on any socket drop” rule — replaced by mandatory - history-version validation on every subscription. -- README's WS section — rewritten as part of the WS v2 cutover. +### 5.1 Admission and continuity + +`GET /ws/subscribe?from_offset=N&era_id=e&recovery_generation=g` requires all +three coordinates. The client claims the history associated with its own state +and the exact application boundary it is ready to execute. Validate the claim +against one coherent read of `(e, g, K, H)` before delivering any input. + +Apply history checks before offset checks: + +| Condition | Response | Client action | +|---|---|---| +| Era differs | `era_changed` | Download and restore a current snapshot. | +| Generation differs | `stale_generation` | Download and restore a current snapshot. | +| `N < K` | `history_unavailable`, with `available_from = K` | Download and restore a current snapshot. | +| `N > H` | `ahead_of_head`, with `live_head = H` | Report the invalid claim; do not wait or silently clamp it. | +| `K <= N <= H` | Admit | Replay inclusively from `N`, then follow new inputs. | + +Policy refusals use a typed error response before any data, followed by close +1008. Successful admission sends `hello` with the admitted recovery generation +and coherent available/head boundaries. Exact response fields and header names +are fixed together with the SDK during the cutover. The era is bound by the +mandatory request claim; every message on an admitted session carries that +session's generation. Refusal metadata describes the history observed when +validating the rejected claim. + +### 5.2 Replay and resource bounds + +Every matching-history offset in `[K, H]` is serveable. For `N < H`, the first +returned input is entry `N`. A request exactly at `H` waits for the next input. +The gold boundary does not gate admission: finalized and soft entries use the +same ordered stream, and advancing finalization requires no client action. + +Read canonical pages from committed valid SQLite history, with a bounded page +size and bounded send queue. Preserve the concurrent-subscriber limit. There +is no total catch-up event cap: neither snapshot cadence nor download/restore +time guarantees a backlog below 50,000 inputs, and refetching can return the +same snapshot repeatedly. Memory and concurrency remain bounded independently +of total history depth. A replica must process faster than ongoing production +to reach the tip; another transport cannot remove that capacity requirement. + +The client verifies that each event's offset equals its application's current +count, applies the input, and advances that count. It resumes from the next +unapplied input, not the last received network message. A client that persists +its replica must preserve the corresponding history identity with that state. +An offset mismatch is a continuity failure; skipping an input or jumping to a +suggested live head would corrupt the replica. + +### 5.3 Snapshot-to-tip walkthrough + +1. Download an artifact with `(era=A, generation=7, count=100)` and restore it. +2. Subscribe with `(A, 7, from_offset=100)`. If the head is 105, receive entries + `100..104`, including any entries already finalized. +3. Continue on the same socket when input 105 commits; no transport handoff or + special catch-up transition is required. +4. After an ordinary disconnect, reconnect using the state's saved history + version and actual next-input count. Clean restart preserves that version. +5. If recovery changes the generation, or a rebuild changes the era, the old + claim is rejected before data. Discard the replica, download a current + snapshot, and repeat. This also handles recovery between steps 1 and 2. + +### 5.4 Events and wire format + +Keep the existing denormalized user-op/direct-input context: nonce, fee, safe +block, batch nonce, input index, block timestamp, and transaction hash where +applicable. Replace physical rowids with canonical next-input coordinates. +JSON text messages share a tagged SDK/server enum containing input events, +`hello`, and typed `error` responses. A live-transition message, frame/batch +boundary events, or best-effort invalidation message requires a concrete +consumer need; the basic replay loop and reconnect rules do not rely on them. + +## 6. Compatibility boundary + +The cutover changes snapshot metadata and the subscription contract together +with the SDK and replica harness. Mandatory history claims and canonical +inclusive offsets replace the optional physical-cursor subscription. Remove +the total catch-up rejection and its suggestion to skip directly to the live +head. An ordinary disconnect permits a same-version resume; recovery requires +a new snapshot in the initial client workflow. + +Keep the deployed README truthful until implementation lands. No intermediate +generation-aware physical-rowid API is needed. The storage foundation alone +does not close the consumer invalidation finding. ## 7. Ordered implementation handoff -> **At-risk dependency:** Bart's 2026-07-28 -> confirmation covers the scalar generation contract only. The `EraId` leg — -> changed-era rejection and current-era bootstrap behavior — is explicitly -> unconfirmed by the consumer. The durable schema slice is cheap to carry, -> but treat the era semantics as provisional in every wire-projection step -> below: the wire form is the part that cannot be cheaply migrated once a -> consumer depends on it, so it must not ship ahead of Bart's review. - -Track 3 owns the remaining consumer-facing history/feed protocol. The -storage/execution foundation in §3.1 is complete. The current public rowid -contract remains unchanged until the API cutover lands as one deployable -protocol boundary. - -1. **Resolve the consumer decision gates.** With Bart, settle the `/inputs` - representation and the current-era post-cockroach bootstrap artifact/API. - Decide future-offset behavior before WS v2 and replay authentication/rate - limits before public exposure. Boundary events remain excluded unless a - consumer demonstrates a need. Track 3 owns the consumer-visible bootstrap - contract; Track 6 owns the dump/image representation it may serve. -2. **Add the typed history read/API foundation.** Define shared history-claim, - page, and policy-error types. Logical boundaries use `EraId`, - `RecoveryGeneration`, and `ExecutedInputCount`, not interchangeable raw - `u64` cursors. Back them with one internally consistent SQLite read of - `(e, g, K, H, G)`, canonical inclusive pagination through - `executed_inputs`, and raw safe-input pagination with provenance. -3. **Implement HTTP bootstrap and finalized replay.** Add - `GET /history-version`, `/inputs`, and `/l2-txs`, including era-tagged pages, - the gold-boundary query, typed below-`K` bootstrap errors, and - pagination/provenance/forced-cascade immutability tests. -4. **Cut `/ws/subscribe` over to v2.** Require - `(EraId, RecoveryGeneration, ExecutedInputCount)`, validate the claim before - admission, serve canonical-coordinate soft history for every admitted offset - at or above `G`, emit the typed hello/control/error vocabulary, carry - `RecoveryGeneration` on every response, and remove the - physical-rowid/string-close contract. Update the SDK and harness in the same - cutover. -5. **Prove and close the feature.** Replace the behavior-pinning E2Es with - stale-generation, era-change, below-base, logical-suffix-reuse, - pagination-hole, and replay-to-live race cases; rewrite the README; graduate - the normative protocol text; remeasure submit-to-matching-WS-event latency; - then mark the WS invalidation-contract finding fixed in the register. - -Steps 2 and the internal part of 3 can begin before the consumer questions -close. Do not freeze the public below-`K` recovery recipe or `/inputs` response -shape until the relevant questions do. Feed output must come from committed -valid SQLite history and match the requested `HistoryVersion`; it does not -depend on a global runtime actor. - -## 8. Decision gates and revisit triggers - -1. **Boundary events:** revisit if a consumer needs frame/batch boundaries the - denormalized rows cannot express. -2. **`/inputs` shape:** confirm how Bart reconciles raw L1 order against feed - order and settlement. -3. **Post-cockroach bootstrap:** select the current-era snapshot/artifact and - API a consumer uses when the old era's ordered history is unavailable. -4. **Future offsets:** decide whether `from_offset > H` waits for the head or - fails with a typed ahead-of-head response. It does not affect the settled - `X`-means-next-input boundary. -5. **Access policy/limits:** decide whether replay remains - internal/network-restricted or needs application authentication, and set - rate limits before public exposure. - -Questions 2 and 3 shape step 3 and need Bart's review. Question 4 gates the WS -v2 contract. Question 5 gates public exposure. Question 1 is explicitly -non-blocking unless a consumer brings a concrete requirement. - -## 9. Review remarks (non-normative) - -- Cockroach recovery cannot currently supply historical user ops as ordered L2 - transactions from genesis. Therefore finalized replay stability is scoped to - an era. The new era retains the recovered application's absolute count as - `K`; a changed history version detects the boundary, and offsets below `K` - fail honestly rather than fabricating the missing feed. -- `EraId` replaces the earlier `instance_id` name because it describes an - externally visible history era, not a process or machine instance. It is - UUIDv4; the first-setup timestamp remains separate metadata. -- The reference scheduler count audit is landed: successful directs and user - ops share one checked typed boundary, overdue-direct ordering is preserved, - and `AppError` is fatal rather than skipped. The durable per-input mapping is - also landed; the public feed cutover is not. +The [coordination roadmap](2026-07-coordination-tracks.md) +owns PR sequencing. Implement two review boundaries: + +1. **Internal read and snapshot foundation (implemented).** Define typed history claims, + canonical pages, snapshot metadata, and policy errors. Read `(e, g, K, H)` + coherently, paginate inclusively through `executed_inputs`, and acquire the + snapshot's count/version with its artifact lease. Preserve physical cursors + for internal catch-up and recovery. Keep intermediate helpers internal. + The unused internal canonical reader has a scoped non-test dead-code + expectation until the next step gives it a runtime caller; remove that + expectation when connecting WS. +2. **Coordinated consumer cutover (next).** Project snapshot metadata over HTTP and + require `(EraId, RecoveryGeneration, ExecutedInputCount)` on WS. Admit the + entire available history range, remove the total catch-up cap, and implement + typed refusals and fresh-snapshot remediation. Update SDK, replica harness, + cache validators, and consumer documentation in the same deployable change. + Fix the exact metadata/error serialization as part of that shared contract. + +Required evidence belongs with the change that owns the behavior: + +- Coherent metadata and canonical pagination across physical-row holes, + envelope/padding rows, nonzero era bases, and replacement suffixes. +- Snapshot bytes, count, and version remain associated through transfer and + cache reuse; restored application count agrees with response metadata. +- Clean restart and ordinary reconnect resume without rebootstrap. +- Stale generation is refused before any data; replacement inputs can reuse + canonical counts without preserving the invalidated replica. +- A changed era is detected even with the same numeric generation/count, and + cache validators differ for rebuilt artifacts at the same inclusion block. +- Below-base and ahead-of-head claims receive their typed errors; a claim at + the head waits normally, and a valid claim below gold is admitted. +- More than 50,000 inputs after the latest snapshot, including direct-heavy + history, remain replayable with bounded pages and queues. +- Writes during download/restoration and replay-to-live delivery cause no gaps; + recovery between snapshot acquisition and subscription forces rebootstrap. +- Subscriber limits, disconnect cleanup, snapshot lease lifetime, and watchdog + finalized comparison behavior remain correct. + +Remeasure submit-to-matching-WS-event latency, rewrite the README, graduate the +normative protocol text into `docs/protocol/`, and close the register's WS +invalidation-contract finding only after the cutover and acceptance evidence. +Feed output comes from committed valid SQLite history matching the admitted +history version; it does not depend on a global runtime actor. + +## 8. Revisit triggers + +- **Archive or historical queries:** evaluate dedicated HTTP replay endpoints + when a consumer needs capabilities beyond snapshot-to-tip replication. +- **Recovery cost:** consider retaining a known-stable client checkpoint when + repeated full snapshot restoration is a measured problem. +- **Snapshot transport:** add resumable transfer or another artifact packaging + only when actual application size/layout and clients require it. +- **Extra stream controls:** add live/frame/batch boundary events only for an + identified consumer operation that cannot use the existing event context. +- **Access boundary:** authentication and public rate-limit policy require a + separate decision if egress is exposed beyond operator infrastructure. +- **Runtime lifecycle:** re-evaluate session fencing if recovery can change + history inside an admitted process or multiple writers are introduced. diff --git a/docs/review/register.md b/docs/review/register.md index c30330ae..815c0409 100644 --- a/docs/review/register.md +++ b/docs/review/register.md @@ -49,7 +49,9 @@ remaining dated ledgers stay valid. log. 6. **WS session hygiene** — a mid-session transient read error tears down with no close frame; a beyond-head `from_offset` idles forever (currently - e2e-pinned as intended — decide the contract, then re-pin). + e2e-pinned as intended). The accepted + [Track 3 contract](../plans/2026-07-track3-feed-replay-design.md#51-admission-and-continuity) + chooses a typed ahead-of-head error; API implementation and re-pinning remain open. 7. **WS invalidation/rollback contract** — `/ws/subscribe` still pages by physical rowid with no `HistoryVersion` claim, so a cursor-resumed subscriber silently keeps invalidated rows across recovery. Interim diff --git a/docs/snapshots/lifecycle.md b/docs/snapshots/lifecycle.md index 45654e48..2182b349 100644 --- a/docs/snapshots/lifecycle.md +++ b/docs/snapshots/lifecycle.md @@ -130,6 +130,11 @@ The split between **pending** and **finalized** mirrors the sequencer's optimism: a batch closes off-chain (soft) → its snapshot is *pending*; the batch lands safe on L1 → its snapshot is *promoted* to finalized. +Egress lease acquisition captures `HistoryVersion` in the same transaction as +the selected artifact and its canonical count. The returned metadata remains +associated with those bytes even if another snapshot is promoted afterward. +HTTP projection of this identity belongs to the Track 3 consumer cutover. + The storage half lives in `storage/snapshot_dumps.rs` (SQLite only — no filesystem); the lane half in `ingress/inclusion_lane/snapshot.rs` + `dump_info.rs` (drives the trait and FS work). That split is load-bearing for diff --git a/sequencer-core/src/history.rs b/sequencer-core/src/history.rs index 6b38e646..3125aa21 100644 --- a/sequencer-core/src/history.rs +++ b/sequencer-core/src/history.rs @@ -136,6 +136,64 @@ pub struct HistoryVersion { pub recovery_generation: RecoveryGeneration, } +/// The history a consumer holds and the next application input it can execute. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct HistoryClaim { + pub version: HistoryVersion, + pub next_input: ExecutedInputCount, +} + +/// One coherent view of the locally available canonical history. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct HistoryBounds { + pub version: HistoryVersion, + pub available_from: ExecutedInputCount, + pub head: ExecutedInputCount, +} + +impl HistoryBounds { + /// Validate identity before position: equal counts cannot resume a different + /// history. Every count in the inclusive range is admissible; `head` waits + /// for the next input. + pub fn validate(&self, claim: HistoryClaim) -> Result<(), HistoryPolicyError> { + assert!( + self.available_from <= self.head, + "available history base exceeds its head" + ); + if claim.version.era_id != self.version.era_id { + return Err(HistoryPolicyError::EraChanged { + current: self.version, + }); + } + if claim.version.recovery_generation != self.version.recovery_generation { + return Err(HistoryPolicyError::StaleGeneration { + current: self.version, + }); + } + if claim.next_input < self.available_from { + return Err(HistoryPolicyError::HistoryUnavailable { + available_from: self.available_from, + }); + } + if claim.next_input > self.head { + return Err(HistoryPolicyError::AheadOfHead { head: self.head }); + } + Ok(()) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Error)] +pub enum HistoryPolicyError { + #[error("history era changed")] + EraChanged { current: HistoryVersion }, + #[error("history generation changed")] + StaleGeneration { current: HistoryVersion }, + #[error("requested input precedes locally available history")] + HistoryUnavailable { available_from: ExecutedInputCount }, + #[error("requested input is ahead of the history head")] + AheadOfHead { head: ExecutedInputCount }, +} + #[cfg(test)] mod tests { use super::*; @@ -181,4 +239,111 @@ mod tests { Some(ExecutedInputCount::new(12)) ); } + + fn history_bounds(base: u64, head: u64) -> HistoryBounds { + HistoryBounds { + version: HistoryVersion { + era_id: EraId::from_bytes(CANONICAL_BYTES).unwrap(), + recovery_generation: RecoveryGeneration::new(4), + }, + available_from: ExecutedInputCount::new(base), + head: ExecutedInputCount::new(head), + } + } + + #[test] + fn history_claim_checks_era_before_generation_and_position() { + let bounds = history_bounds(41, 45); + let mut other_era = CANONICAL_BYTES; + other_era[0] ^= 1; + let other_era = EraId::from_bytes(other_era).unwrap(); + for (generation, next_input) in [(4, 43), (3, 0), (u64::MAX, u64::MAX)] { + assert_eq!( + bounds.validate(HistoryClaim { + version: HistoryVersion { + era_id: other_era, + recovery_generation: RecoveryGeneration::new(generation), + }, + next_input: ExecutedInputCount::new(next_input), + }), + Err(HistoryPolicyError::EraChanged { + current: bounds.version, + }) + ); + } + } + + #[test] + fn history_claim_requires_equal_generation_before_checking_position() { + let bounds = history_bounds(41, 45); + for generation in [0, 3, 5, u64::MAX] { + for next_input in [0, 43, u64::MAX] { + assert_eq!( + bounds.validate(HistoryClaim { + version: HistoryVersion { + recovery_generation: RecoveryGeneration::new(generation), + ..bounds.version + }, + next_input: ExecutedInputCount::new(next_input), + }), + Err(HistoryPolicyError::StaleGeneration { + current: bounds.version, + }) + ); + } + } + } + + #[test] + fn history_claim_rejects_unavailable_and_future_counts() { + let bounds = history_bounds(41, 45); + for next_input in [0, 40, 46, u64::MAX] { + let expected = if next_input < 41 { + HistoryPolicyError::HistoryUnavailable { + available_from: bounds.available_from, + } + } else { + HistoryPolicyError::AheadOfHead { head: bounds.head } + }; + assert_eq!( + bounds.validate(HistoryClaim { + version: bounds.version, + next_input: ExecutedInputCount::new(next_input), + }), + Err(expected) + ); + } + } + + #[test] + fn history_claim_accepts_the_full_inclusive_range_without_a_depth_cap() { + for (base, head, next_input) in [ + (0, 0, 0), + (41, 45, 41), + (41, 45, 43), + (41, 45, 45), + (0, 100_001, 0), + (i64::MAX as u64, i64::MAX as u64 + 1, i64::MAX as u64 + 1), + (u64::MAX, u64::MAX, u64::MAX), + ] { + let bounds = history_bounds(base, head); + assert_eq!( + bounds.validate(HistoryClaim { + version: bounds.version, + next_input: ExecutedInputCount::new(next_input), + }), + Ok(()) + ); + } + } + + #[test] + #[should_panic(expected = "available history base exceeds its head")] + fn incoherent_history_bounds_fail_loud() { + let bounds = history_bounds(42, 41); + let _ = bounds.validate(HistoryClaim { + version: bounds.version, + next_input: ExecutedInputCount::new(41), + }); + } } diff --git a/sequencer/src/egress/l2_tx_feed/mod.rs b/sequencer/src/egress/l2_tx_feed/mod.rs index da04b8ac..12ede2c9 100644 --- a/sequencer/src/egress/l2_tx_feed/mod.rs +++ b/sequencer/src/egress/l2_tx_feed/mod.rs @@ -19,7 +19,7 @@ use tokio::sync::mpsc; use crate::runtime::process_lock::spawn_blocking_with_lock; use crate::runtime::shutdown::{RuntimeScope, abort_terminal}; -use crate::storage::{OrderedL2TxRow, Storage}; +use crate::storage::{L2TxContext, Storage}; /// Best-effort extraction of a panic payload's message for fault causes. fn panic_message(payload: &dyn std::any::Any) -> &str { @@ -260,18 +260,17 @@ fn run_subscription( return Ok(()); } - next_offset = row.offset(); - let event = match row { - OrderedL2TxRow::UserOp { - offset, + next_offset = row.offset; + let offset = row.offset; + let event = match row.context { + L2TxContext::UserOp { tx, nonce, safe_block, batch_nonce, .. } => BroadcastTxMessage::from_user_op(offset, tx, nonce, safe_block, batch_nonce), - OrderedL2TxRow::DirectInput { - offset, + L2TxContext::DirectInput { tx, input_index, batch_nonce, diff --git a/sequencer/src/storage/egress.rs b/sequencer/src/storage/egress.rs index 71d20a05..26615416 100644 --- a/sequencer/src/storage/egress.rs +++ b/sequencer/src/storage/egress.rs @@ -3,12 +3,11 @@ //! Egress reader: ordered-L2-tx queries used by the WS feed and catch-up replay. //! -//! Read-only — every method here either pages the `valid_sequenced_l2_txs` view -//! or counts over it. The view encapsulates the exclusion of invalidated batches -//! so callers don't repeat the filter. +//! Physical replay and canonical history use their respective `valid_*` views; +//! payload and provenance decoding is shared between both coordinates. use alloy_primitives::{Address, B256}; -use rusqlite::{Result, params}; +use rusqlite::{Result, Row, params}; use super::Storage; use super::convert::{i64_to_u32, i64_to_u64, saturating_query_bound}; @@ -16,62 +15,55 @@ use super::queries::decode_l2_tx_row; use sequencer_core::history::ExecutedInputCount; use sequencer_core::l2_tx::{DirectInput, SequencedL2Tx, ValidUserOp}; -/// One persisted L2 transaction and the ordering context of its covering frame. +#[cfg_attr( + not(test), + expect( + dead_code, + reason = "Internal canonical reads await the coordinated WS cutover." + ) +)] +mod canonical; + +/// Application input and persisted context shared by both replay coordinates. #[derive(Debug, Clone)] -pub(crate) enum OrderedL2TxRow { +pub(crate) enum L2TxContext { UserOp { - offset: u64, tx: ValidUserOp, nonce: u32, safe_block: u64, batch_nonce: u64, - executed_input_offset: Option, }, DirectInput { - offset: u64, tx: DirectInput, input_index: u64, safe_block: u64, batch_nonce: u64, block_timestamp: u64, transaction_hash: B256, - executed_input_offset: Option, }, } -impl OrderedL2TxRow { - pub(crate) fn offset(&self) -> u64 { - match self { - Self::UserOp { offset, .. } | Self::DirectInput { offset, .. } => *offset, - } - } +/// Physical replay row, including rows that do not execute in the application. +#[derive(Debug, Clone)] +pub(crate) struct OrderedL2TxRow { + pub(crate) offset: u64, + pub(crate) executed_input_offset: Option, + pub(crate) context: L2TxContext, +} +impl OrderedL2TxRow { fn into_replay_row(self) -> ReplayL2TxRow { - match self { - Self::UserOp { - offset, - tx, - safe_block, - executed_input_offset, - .. - } => ReplayL2TxRow { - db_offset: offset, - tx: SequencedL2Tx::UserOp(tx), - frame_safe_block: safe_block, - executed_input_offset, - }, - Self::DirectInput { - offset, - tx, - safe_block, - executed_input_offset, - .. - } => ReplayL2TxRow { - db_offset: offset, - tx: SequencedL2Tx::Direct(tx), - frame_safe_block: safe_block, - executed_input_offset, - }, + let (tx, frame_safe_block) = match self.context { + L2TxContext::UserOp { tx, safe_block, .. } => (SequencedL2Tx::UserOp(tx), safe_block), + L2TxContext::DirectInput { tx, safe_block, .. } => { + (SequencedL2Tx::Direct(tx), safe_block) + } + }; + ReplayL2TxRow { + db_offset: self.offset, + tx, + frame_safe_block, + executed_input_offset: self.executed_input_offset, } } } @@ -167,44 +159,7 @@ impl Storage { saturating_query_bound(offset), saturating_query_bound(limit) ], - |row| { - let db_offset: i64 = row.get(0)?; - let tx = decode_l2_tx_row( - row.get(1)?, - row.get(2)?, - row.get(3)?, - row.get(4)?, - row.get(5)?, - row.get(6)?, - ); - // Non-NULL for every sequenced row: batches and frames exist before - // anything can be sequenced into them. - let safe_block = i64_to_u64(row.get(7)?); - let batch_nonce = i64_to_u64(row.get(8)?); - let executed_input_offset = row - .get::<_, Option>(13)? - .map(|value| ExecutedInputCount::new(i64_to_u64(value))); - match tx { - SequencedL2Tx::UserOp(tx) => Ok(OrderedL2TxRow::UserOp { - offset: i64_to_u64(db_offset), - tx, - nonce: i64_to_u32(row.get(10)?), - safe_block, - batch_nonce, - executed_input_offset, - }), - SequencedL2Tx::Direct(tx) => Ok(OrderedL2TxRow::DirectInput { - offset: i64_to_u64(db_offset), - tx, - input_index: i64_to_u64(row.get(9)?), - safe_block, - batch_nonce, - block_timestamp: i64_to_u64(row.get(11)?), - transaction_hash: B256::from_slice(row.get::<_, Vec>(12)?.as_slice()), - executed_input_offset, - }), - } - }, + decode_ordered_l2_tx_row, )?; rows.collect::>>() } @@ -254,3 +209,39 @@ impl Storage { Ok(i64_to_u64(value)) } } + +fn decode_ordered_l2_tx_row(row: &Row<'_>) -> Result { + let tx = decode_l2_tx_row( + row.get(1)?, + row.get(2)?, + row.get(3)?, + row.get(4)?, + row.get(5)?, + row.get(6)?, + ); + let safe_block = i64_to_u64(row.get(7)?); + let batch_nonce = i64_to_u64(row.get(8)?); + let context = match tx { + SequencedL2Tx::UserOp(tx) => L2TxContext::UserOp { + tx, + nonce: i64_to_u32(row.get(10)?), + safe_block, + batch_nonce, + }, + SequencedL2Tx::Direct(tx) => L2TxContext::DirectInput { + tx, + input_index: i64_to_u64(row.get(9)?), + safe_block, + batch_nonce, + block_timestamp: i64_to_u64(row.get(11)?), + transaction_hash: B256::from_slice(row.get::<_, Vec>(12)?.as_slice()), + }, + }; + Ok(OrderedL2TxRow { + offset: i64_to_u64(row.get(0)?), + executed_input_offset: row + .get::<_, Option>(13)? + .map(|value| ExecutedInputCount::new(i64_to_u64(value))), + context, + }) +} diff --git a/sequencer/src/storage/egress/canonical.rs b/sequencer/src/storage/egress/canonical.rs new file mode 100644 index 00000000..e84e07ff --- /dev/null +++ b/sequencer/src/storage/egress/canonical.rs @@ -0,0 +1,157 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +//! Version-checked canonical pages, read with their boundaries in one transaction. + +use rusqlite::{Connection, Transaction, params}; +use sequencer_core::history::{ + ExecutedInputCount, HistoryBounds, HistoryClaim, HistoryPolicyError, +}; + +use super::{L2TxContext, decode_ordered_l2_tx_row}; +use crate::storage::Storage; +use crate::storage::convert::{saturating_query_bound, u64_to_i64}; +use crate::storage::history::{next_executed_input_count_in, query_history_state}; + +#[derive(Debug)] +pub(crate) struct CanonicalHistoryRow { + pub(crate) offset: ExecutedInputCount, + pub(crate) context: L2TxContext, +} + +#[derive(Debug)] +pub(crate) struct CanonicalHistoryPage { + pub(crate) bounds: HistoryBounds, + pub(crate) rows: Vec, + pub(crate) next: HistoryClaim, +} + +#[derive(Debug, thiserror::Error)] +pub(crate) enum HistoryReadError { + #[error(transparent)] + Policy(#[from] HistoryPolicyError), + #[error("reading canonical history: {0}")] + Storage(#[from] rusqlite::Error), +} + +impl Storage { + pub(crate) fn history_bounds(&mut self) -> rusqlite::Result { + self.read(|tx| history_bounds_in(tx)) + } + + /// Validate before reading inputs, including for empty pages. An accepted + /// count at the head waits; a missing row before that head is an invariant fault. + pub(crate) fn canonical_history_page( + &mut self, + claim: HistoryClaim, + limit: usize, + ) -> Result { + self.read(|tx| { + let bounds = history_bounds_in(tx)?; + match bounds.validate(claim) { + Ok(()) => canonical_page_in(tx, bounds, claim.next_input, limit).map(Ok), + Err(error) => Ok(Err(error)), + } + })? + .map_err(HistoryReadError::Policy) + } +} + +fn history_bounds_in(conn: &Connection) -> rusqlite::Result { + let state = query_history_state(conn)?; + let available_from = ExecutedInputCount::new( + state + .base_executed_input_count + .expect("application history base is unbound outside rebuild fill"), + ); + Ok(HistoryBounds { + version: state.version, + available_from, + head: next_executed_input_count_in(conn)?, + }) +} + +fn canonical_page_in( + tx: &Transaction<'_>, + bounds: HistoryBounds, + from: ExecutedInputCount, + limit: usize, +) -> rusqlite::Result { + let limit = u64::try_from(limit).expect("page size fits u64"); + let expected_len = limit.min(bounds.head.get() - from.get()); + let mut rows = Vec::new(); + // H can be i64::MAX + 1, the boundary after the last representable row. + // An empty-at-head request must never clamp back onto that last input. + if expected_len > 0 { + const SQL: &str = " + SELECT + s.sequenced_l2_tx_offset, + CASE WHEN s.user_op_pos_in_frame IS NOT NULL THEN 0 ELSE 1 END, + CASE WHEN s.user_op_pos_in_frame IS NOT NULL THEN u.sender ELSE d.sender END, + CASE WHEN s.user_op_pos_in_frame IS NOT NULL THEN u.data ELSE NULL END, + CASE WHEN s.user_op_pos_in_frame IS NOT NULL THEN f.fee ELSE NULL END, + CASE WHEN s.safe_input_index IS NOT NULL THEN d.payload ELSE NULL END, + CASE WHEN s.safe_input_index IS NOT NULL THEN d.block_number ELSE NULL END, + f.safe_block, + b.nonce, + s.safe_input_index, + CASE WHEN s.user_op_pos_in_frame IS NOT NULL THEN u.nonce ELSE NULL END, + CASE WHEN s.safe_input_index IS NOT NULL THEN d.block_timestamp ELSE NULL END, + CASE WHEN s.safe_input_index IS NOT NULL THEN d.transaction_hash ELSE NULL END, + s.executed_input_offset + FROM valid_executed_inputs s + LEFT JOIN user_ops u + ON u.batch_index = s.batch_index + AND u.frame_in_batch = s.frame_in_batch + AND u.pos_in_frame = s.user_op_pos_in_frame + LEFT JOIN frames f + ON f.batch_index = s.batch_index AND f.frame_in_batch = s.frame_in_batch + LEFT JOIN safe_inputs d ON d.safe_input_index = s.safe_input_index + LEFT JOIN batches b ON b.batch_index = s.batch_index + WHERE s.executed_input_offset >= ?1 + ORDER BY s.executed_input_offset + LIMIT ?2 + "; + let mut stmt = tx.prepare_cached(SQL)?; + let mapped = stmt.query_map( + params![u64_to_i64(from.get()), saturating_query_bound(expected_len)], + decode_ordered_l2_tx_row, + )?; + let mut expected = from; + for row in mapped { + let row = row?; + let offset = row + .executed_input_offset + .expect("canonical history row has no execution attribution"); + assert_eq!( + offset, expected, + "canonical history page has an attribution gap" + ); + rows.push(CanonicalHistoryRow { + offset, + context: row.context, + }); + expected = expected + .checked_next() + .expect("canonical input count overflow"); + } + assert_eq!( + rows.len() as u64, + expected_len, + "canonical history page ended before its recorded head" + ); + } + Ok(CanonicalHistoryPage { + bounds, + next: HistoryClaim { + version: bounds.version, + next_input: from + .checked_add(expected_len) + .expect("page ends at or before head"), + }, + rows, + }) +} + +#[cfg(test)] +mod tests; diff --git a/sequencer/src/storage/egress/canonical/tests.rs b/sequencer/src/storage/egress/canonical/tests.rs new file mode 100644 index 00000000..580a28d8 --- /dev/null +++ b/sequencer/src/storage/egress/canonical/tests.rs @@ -0,0 +1,432 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +use std::time::SystemTime; + +use alloy_primitives::{B256, Signature}; +use sequencer_core::user_op::{SignedUserOp, UserOp}; +use tokio::sync::oneshot; + +use super::*; +use crate::ingress::inclusion_lane::{IncludedUserOp, PendingUserOp}; +use crate::storage::test_helpers::{ + SENDER_A, SENDER_B, default_protocol_timing, local_batch_payload, pin_test_deployment_identity, + temp_db, +}; +use crate::storage::{ + DirectInputExecution, FrontierMode, IngestedSafeInput, LifecycleCommand, SafeInputRange, + StoredSafeInput, +}; + +fn included(nonce: u32, offset: u64, payload: u8) -> IncludedUserOp { + let (respond_to, _response) = oneshot::channel(); + IncludedUserOp { + pending: PendingUserOp { + signed: SignedUserOp { + sender: SENDER_B, + signature: Signature::test_signature(), + user_op: UserOp { + nonce, + max_fee: u16::MAX, + data: vec![payload].into(), + }, + }, + respond_to, + received_at: SystemTime::now(), + }, + executed_input_offset: ExecutedInputCount::new(offset), + } +} + +fn claim(bounds: HistoryBounds, next: u64) -> HistoryClaim { + HistoryClaim { + version: bounds.version, + next_input: ExecutedInputCount::new(next), + } +} + +fn seed_aging_tip(storage: &mut Storage) { + pin_test_deployment_identity(storage, SENDER_A); + let mut head = storage + .initialize_open_state(0, SafeInputRange::empty_at(0)) + .unwrap(); + storage + .append_executed_user_ops_chunk(&mut head, &[included(0, 0, 0xaa)]) + .unwrap(); + storage.close_frame_and_batch(&mut head, 0).unwrap(); + storage + .append_executed_user_ops_chunk(&mut head, &[included(1, 1, 0xbb)]) + .unwrap(); + storage + .append_safe_inputs(1_500, &[], SENDER_A, &default_protocol_timing()) + .unwrap(); +} + +#[test] +fn canonical_pages_are_inclusive_and_preserve_context_without_batch_envelopes() { + let db = temp_db("canonical-page-context"); + let mut storage = Storage::open(&db.path).unwrap(); + pin_test_deployment_identity(&mut storage, SENDER_A); + let mut head = storage + .initialize_open_state(0, SafeInputRange::empty_at(0)) + .unwrap(); + let first_fee = head.frame_fee; + storage + .append_executed_user_ops_chunk(&mut head, &[included(7, 0, 0x11)]) + .unwrap(); + storage.close_frame_and_batch(&mut head, 0).unwrap(); + let envelope = local_batch_payload(&mut storage, 0); + let transaction_hash = B256::repeat_byte(0x42); + storage + .append_ingested_safe_inputs_with_timestamp( + 10, + 100, + &[ + IngestedSafeInput { + sender: SENDER_B, + payload: vec![0x22], + block_number: 10, + block_timestamp: 100, + transaction_hash, + }, + IngestedSafeInput { + sender: SENDER_A, + payload: envelope, + block_number: 10, + block_timestamp: 100, + transaction_hash: B256::repeat_byte(0x43), + }, + ], + SENDER_A, + &default_protocol_timing(), + FrontierMode::Populate, + ) + .unwrap(); + storage + .close_frame_only_with_executions( + &mut head, + 10, + SafeInputRange::new(0, 2), + &[DirectInputExecution { + safe_input_index: 0, + executed_input_offset: ExecutedInputCount::new(1), + }], + None, + ) + .unwrap(); + storage + .append_executed_user_ops_chunk(&mut head, &[included(8, 2, 0x33)]) + .unwrap(); + + let bounds = storage.history_bounds().unwrap(); + assert_eq!(bounds.available_from, ExecutedInputCount::ZERO); + assert_eq!(bounds.head, ExecutedInputCount::new(3)); + assert_eq!(storage.ordered_l2_txs_page_from(0, 10).unwrap().len(), 4); + let first = storage.canonical_history_page(claim(bounds, 0), 2).unwrap(); + assert_eq!(first.bounds, bounds); + assert_eq!(first.next, claim(bounds, 2)); + assert_eq!(first.rows.len(), 2); + assert_eq!(first.rows[0].offset, ExecutedInputCount::ZERO); + match &first.rows[0].context { + L2TxContext::UserOp { + tx, + nonce, + safe_block, + batch_nonce, + } => { + assert_eq!(tx.sender, SENDER_B); + assert_eq!(tx.data, vec![0x11]); + assert_eq!(tx.fee, first_fee); + assert_eq!((*nonce, *safe_block, *batch_nonce), (7, 0, 0)); + } + other => panic!("expected user op, got {other:?}"), + } + assert_eq!(first.rows[1].offset, ExecutedInputCount::new(1)); + match &first.rows[1].context { + L2TxContext::DirectInput { + tx, + input_index, + safe_block, + batch_nonce, + block_timestamp, + transaction_hash: actual_hash, + } => { + assert_eq!(tx.sender, SENDER_B); + assert_eq!(tx.payload, vec![0x22]); + assert_eq!(tx.block_number, 10); + assert_eq!( + (*input_index, *safe_block, *batch_nonce, *block_timestamp), + (0, 10, 1, 100) + ); + assert_eq!(*actual_hash, transaction_hash); + } + other => panic!("expected direct input, got {other:?}"), + } + let last = storage.canonical_history_page(first.next, 2).unwrap(); + assert_eq!(last.rows.len(), 1); + assert_eq!(last.rows[0].offset, ExecutedInputCount::new(2)); + match &last.rows[0].context { + L2TxContext::UserOp { + tx, + nonce, + safe_block, + batch_nonce, + } => { + assert_eq!(tx.data, vec![0x33]); + assert_eq!(tx.fee, head.frame_fee); + assert_eq!((*nonce, *safe_block, *batch_nonce), (8, 10, 1)); + } + other => panic!("expected user op, got {other:?}"), + } + assert_eq!(last.next, claim(bounds, 3)); + let tail = storage.canonical_history_page(last.next, 2).unwrap(); + assert!(tail.rows.is_empty()); + assert_eq!(tail.next, last.next); + let zero = storage.canonical_history_page(claim(bounds, 1), 0).unwrap(); + assert!(zero.rows.is_empty()); + assert_eq!(zero.next, claim(bounds, 1)); +} + +#[test] +fn recovery_refuses_old_claims_and_reuses_canonical_offsets_across_physical_holes() { + let db = temp_db("canonical-page-recovery"); + let mut storage = Storage::open(&db.path).unwrap(); + seed_aging_tip(&mut storage); + let before = storage.history_bounds().unwrap(); + assert_eq!(before.head, ExecutedInputCount::new(2)); + + assert_eq!(storage.recover_aging_tip(1_200).unwrap(), vec![1]); + let recovered = storage.history_bounds().unwrap(); + assert_eq!(recovered.version.era_id, before.version.era_id); + assert_eq!(recovered.version.recovery_generation.get(), 1); + assert_eq!(recovered.head, ExecutedInputCount::new(1)); + for limit in [0, 2] { + assert!(matches!( + storage.canonical_history_page(claim(before, 2), limit), + Err(HistoryReadError::Policy(HistoryPolicyError::StaleGeneration { current })) + if current == recovered.version + )); + } + assert!( + storage + .canonical_history_page(claim(recovered, 1), 2) + .unwrap() + .rows + .is_empty() + ); + let mut head = storage.open_state().unwrap().unwrap(); + storage + .append_executed_user_ops_chunk(&mut head, &[included(1, 1, 0xcc)]) + .unwrap(); + let page = storage + .canonical_history_page(claim(recovered, 1), 2) + .unwrap(); + assert_eq!(page.rows.len(), 1); + assert_eq!(page.rows[0].offset, ExecutedInputCount::new(1)); + assert_eq!(page.next.next_input, before.head); + match &page.rows[0].context { + L2TxContext::UserOp { tx, .. } => assert_eq!(tx.data, vec![0xcc]), + other => panic!("expected replacement user op, got {other:?}"), + } + let physical = storage.ordered_l2_txs_page_from(0, 10).unwrap(); + assert_eq!( + physical.iter().map(|row| row.db_offset).collect::>(), + vec![1, 3] + ); + let audit_rows: i64 = storage + .conn + .query_row("SELECT COUNT(*) FROM sequenced_l2_txs", [], |row| { + row.get(0) + }) + .unwrap(); + assert_eq!(audit_rows, 3); +} + +#[test] +fn rebuilt_history_starts_at_its_absolute_base_and_excludes_padding() { + let db = temp_db("canonical-page-rebuild"); + let mut storage = Storage::initialize_for_command(&db.path, LifecycleCommand::Rebuild).unwrap(); + storage + .append_safe_inputs( + 10, + &[ + StoredSafeInput { + sender: SENDER_B, + payload: vec![0xaa], + block_number: 10, + }, + StoredSafeInput { + sender: SENDER_B, + payload: vec![0xbb], + block_number: 10, + }, + ], + SENDER_A, + &default_protocol_timing(), + ) + .unwrap(); + storage.open_recovery_tip(10).unwrap(); + let physical_head = storage.valid_ordered_l2_tx_head().unwrap(); + storage + .insert_initial_finalized_dump(&db._dir.path().join("recovered"), 10, physical_head, 41, 2) + .unwrap(); + let bounds = storage.history_bounds().unwrap(); + assert_eq!(bounds.available_from, ExecutedInputCount::new(41)); + assert_eq!(bounds.head, bounds.available_from); + assert!( + storage + .canonical_history_page(claim(bounds, 41), 10) + .unwrap() + .rows + .is_empty() + ); + assert!( + matches!(storage.canonical_history_page(claim(bounds, 40), 10), + Err(HistoryReadError::Policy(HistoryPolicyError::HistoryUnavailable { available_from })) + if available_from == bounds.available_from) + ); + assert!( + matches!(storage.canonical_history_page(claim(bounds, 42), 10), + Err(HistoryReadError::Policy(HistoryPolicyError::AheadOfHead { head })) + if head == bounds.head) + ); + + let mut head = storage.open_state().unwrap().unwrap(); + storage + .append_executed_user_ops_chunk(&mut head, &[included(0, 41, 0xcc)]) + .unwrap(); + let page = storage + .canonical_history_page(claim(bounds, 41), 10) + .unwrap(); + assert_eq!(page.rows.len(), 1); + assert_eq!(page.rows[0].offset, ExecutedInputCount::new(41)); + assert_eq!(page.next.next_input, ExecutedInputCount::new(42)); + let physical = storage.ordered_l2_txs_page_from(0, 10).unwrap(); + assert_eq!(physical.len(), 3); + assert!( + physical[..2] + .iter() + .all(|row| row.executed_input_offset.is_none()) + ); +} + +#[test] +fn a_deep_backlog_can_be_read_in_small_bounded_pages() { + let db = temp_db("canonical-page-deep-backlog"); + let mut storage = Storage::open(&db.path).unwrap(); + let mut head = storage + .initialize_open_state(0, SafeInputRange::empty_at(0)) + .unwrap(); + let inputs: Vec<_> = (0..50_001) + .map(|offset| included(offset, u64::from(offset), 0xaa)) + .collect(); + storage + .append_executed_user_ops_chunk(&mut head, &inputs) + .unwrap(); + let bounds = storage.history_bounds().unwrap(); + assert_eq!(bounds.head, ExecutedInputCount::new(50_001)); + for from in [0, 25_000, 49_999] { + let page = storage + .canonical_history_page(claim(bounds, from), 2) + .unwrap(); + assert_eq!( + page.rows + .iter() + .map(|row| row.offset.get()) + .collect::>(), + vec![from, from + 1] + ); + assert_eq!(page.next, claim(bounds, from + 2)); + } +} + +#[test] +fn one_read_transaction_keeps_history_identity_and_rows_coherent_during_recovery() { + let db = temp_db("canonical-page-read-snapshot"); + let mut writer = Storage::open(&db.path).unwrap(); + seed_aging_tip(&mut writer); + let mut reader = Storage::open_read_only(&db.path).unwrap(); + let tx = reader.conn.transaction().unwrap(); + let before = history_bounds_in(&tx).unwrap(); + + writer.recover_aging_tip(1_200).unwrap(); + let mut head = writer.open_state().unwrap().unwrap(); + writer + .append_executed_user_ops_chunk(&mut head, &[included(1, 1, 0xcc)]) + .unwrap(); + let retained = canonical_page_in(&tx, before, ExecutedInputCount::new(1), 2).unwrap(); + assert_eq!(retained.bounds, before); + assert_eq!(history_bounds_in(&tx).unwrap(), before); + assert_eq!(retained.rows.len(), 1); + match &retained.rows[0].context { + L2TxContext::UserOp { tx, .. } => assert_eq!(tx.data, vec![0xbb]), + other => panic!("expected old snapshot user op, got {other:?}"), + } + tx.commit().unwrap(); + + let after = reader.history_bounds().unwrap(); + assert_eq!(after.head, before.head); + assert_ne!(after.version, before.version); + let fresh = reader.canonical_history_page(claim(after, 1), 2).unwrap(); + assert_eq!(fresh.bounds, after); + match &fresh.rows[0].context { + L2TxContext::UserOp { tx, .. } => assert_eq!(tx.data, vec![0xcc]), + other => panic!("expected current user op, got {other:?}"), + } +} + +#[test] +fn tail_after_the_largest_sqlite_offset_is_empty_without_clamping() { + let db = temp_db("canonical-page-sqlite-tail"); + let mut storage = Storage::initialize_for_command(&db.path, LifecycleCommand::Rebuild).unwrap(); + storage.open_recovery_tip(0).unwrap(); + let last = i64::MAX as u64; + storage + .insert_initial_finalized_dump(&db._dir.path().join("recovered"), 0, 0, last, 0) + .unwrap(); + let mut head = storage.open_state().unwrap().unwrap(); + storage + .append_executed_user_ops_chunk(&mut head, &[included(0, last, 0xaa)]) + .unwrap(); + let bounds = storage.history_bounds().unwrap(); + assert_eq!(bounds.head, ExecutedInputCount::new(last + 1)); + let page = storage + .canonical_history_page(claim(bounds, last), 1) + .unwrap(); + assert_eq!(page.rows.len(), 1); + assert_eq!(page.rows[0].offset, ExecutedInputCount::new(last)); + let tail = storage + .canonical_history_page(page.next, usize::MAX) + .unwrap(); + assert!(tail.rows.is_empty()); + assert_eq!(tail.next, page.next); +} + +#[test] +#[should_panic(expected = "canonical history page has an attribution gap")] +fn an_interior_mapping_hole_fails_loud() { + let db = temp_db("canonical-page-corrupt-attribution"); + let mut storage = Storage::open(&db.path).unwrap(); + let mut head = storage + .initialize_open_state(0, SafeInputRange::empty_at(0)) + .unwrap(); + storage + .append_executed_user_ops_chunk( + &mut head, + &[ + included(0, 0, 0xaa), + included(1, 1, 0xbb), + included(2, 2, 0xcc), + ], + ) + .unwrap(); + storage + .conn + .execute_batch( + "DROP TRIGGER trg_protect_valid_executed_input_delete;\n\ + DELETE FROM executed_inputs WHERE executed_input_offset = 1;", + ) + .unwrap(); + let bounds = storage.history_bounds().unwrap(); + storage.canonical_history_page(claim(bounds, 0), 3).unwrap(); +} diff --git a/sequencer/src/storage/mod.rs b/sequencer/src/storage/mod.rs index 5beecdee..3d8db2d9 100644 --- a/sequencer/src/storage/mod.rs +++ b/sequencer/src/storage/mod.rs @@ -52,7 +52,7 @@ pub(crate) use convert::is_persistent_storage_error; use std::time::SystemTime; use thiserror::Error; -pub(crate) use egress::OrderedL2TxRow; +pub(crate) use egress::L2TxContext; pub use history::{DirectInputExecution, HistoryState}; pub use lifecycle::{LifecycleCommand, LifecycleError, TerminalFault}; pub use open::Storage; diff --git a/sequencer/src/storage/snapshot_dumps.rs b/sequencer/src/storage/snapshot_dumps.rs index 438cbb71..541a2468 100644 --- a/sequencer/src/storage/snapshot_dumps.rs +++ b/sequencer/src/storage/snapshot_dumps.rs @@ -19,9 +19,9 @@ use std::sync::Arc; use rusqlite::{OptionalExtension, Result, Transaction, params}; use super::convert::{i64_to_u64, u64_to_i64}; -use super::history::{bind_history_base_in, next_executed_input_count_in}; +use super::history::{bind_history_base_in, next_executed_input_count_in, query_history_state}; use super::{Storage, is_persistent_storage_error, is_persistent_storage_open_error}; -use sequencer_core::history::ExecutedInputCount; +use sequencer_core::history::{ExecutedInputCount, HistoryVersion}; /// A row in `dumps`: SQLite primary key plus the on-disk directory. #[derive(Debug, Clone, PartialEq, Eq)] @@ -118,6 +118,8 @@ pub struct LeasedDump { pub prefix: PathBuf, pub l2_tx_index: u64, pub executed_input_count: ExecutedInputCount, + /// History in which this artifact was selected, captured with its lease. + pub history_version: HistoryVersion, pub guard: LeaseGuard, } @@ -281,8 +283,8 @@ impl Storage { self.read(latest_snapshot_in) } - /// Atomically read the finalized snapshot AND lease its dump, returning it - /// bundled with an armed release ([`LeaseGuard`]). Closes the race where a + /// Atomically read the finalized snapshot and history version, and lease + /// its dump, bundled with an armed release ([`LeaseGuard`]). Closes the race where a /// handler reads the row, a promotion + GC delete the dump, and the open /// then fails: the lease is held from the moment of the read. `None` if no /// finalized snapshot exists. `schedule` controls where the (blocking) @@ -300,22 +302,27 @@ impl Storage { let Some(finalized) = finalized_dump_in(tx)? else { return Ok(None); }; + let history_version = query_history_state(tx)?.version; acquire_dump_lease_in(tx, finalized.dump.id)?; - Ok(Some(finalized)) + Ok(Some((finalized, history_version))) })?; Ok(acquired.map( - |FinalizedDump { - dump, - inclusion_block, - l2_tx_index, - executed_input_count, - }| FinalizedLease { + |( + FinalizedDump { + dump, + inclusion_block, + l2_tx_index, + executed_input_count, + }, + history_version, + )| FinalizedLease { inclusion_block, dump: LeasedDump { prefix: dump.prefix, l2_tx_index, executed_input_count, + history_version, // Arm the release only after `Storage::write` has committed // the increment. A failed COMMIT rolls back the lease and // must not schedule a decrement for a lease that never @@ -345,16 +352,23 @@ impl Storage { let Some((dump, l2_tx_index, executed_input_count)) = latest_snapshot_in(tx)? else { return Ok(None); }; + let history_version = query_history_state(tx)?.version; let dump_id = dump.id; acquire_dump_lease_in(tx, dump_id)?; - Ok(Some((dump, l2_tx_index, executed_input_count))) + Ok(Some(( + dump, + l2_tx_index, + executed_input_count, + history_version, + ))) })?; - Ok( - acquired.map(|(dump, l2_tx_index, executed_input_count)| LeasedDump { + Ok(acquired.map( + |(dump, l2_tx_index, executed_input_count, history_version)| LeasedDump { prefix: dump.prefix, l2_tx_index, executed_input_count, + history_version, // See `acquire_finalized_lease`: the guard owns a release only // after the matching increment is durable. guard: LeaseGuard { @@ -363,8 +377,8 @@ impl Storage { schedule, report_persistent_failure, }, - }), - ) + }, + )) } /// Return every row in `dumps`. Used at startup to reconcile @@ -767,12 +781,21 @@ fn path_to_text(path: &Path) -> String { #[cfg(test)] mod tests { + use alloy_primitives::{Address, Signature}; + use sequencer_core::history::RecoveryGeneration; + use sequencer_core::user_op::{SignedUserOp, UserOp}; use std::collections::HashSet; use std::path::PathBuf; use std::sync::Arc; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; + use std::time::SystemTime; + use tokio::sync::oneshot; - use crate::storage::{ExecutedInputCount, LifecycleCommand, Storage, test_helpers::temp_db}; + use crate::ingress::inclusion_lane::{IncludedUserOp, PendingUserOp}; + use crate::storage::{ + ExecutedInputCount, LifecycleCommand, SafeInputRange, Storage, + history::advance_recovery_generation_in, test_helpers::temp_db, + }; use super::{DumpRow, FinalizedDump, FinalizedLease, LeaseGuard, PendingDump}; @@ -1351,6 +1374,135 @@ mod tests { ); } + #[test] + fn snapshot_leases_keep_artifact_count_and_history_after_promotion_and_generation_advance() { + let db = temp_db("snapshot-lease-history"); + let mut storage = Storage::open(db.path.as_str()).expect("open"); + let mut head = storage + .initialize_open_state(0, SafeInputRange::empty_at(0)) + .unwrap(); + let first_id = storage.insert_finalized_dump(&prefix(0), 100, 0).unwrap(); + let first_version = storage.history_state().unwrap().version; + let first_finalized = storage + .acquire_finalized_lease(Arc::new(inline), noop_reporter()) + .unwrap() + .unwrap(); + let first_latest = storage + .acquire_latest_snapshot_lease(Arc::new(inline), noop_reporter()) + .unwrap() + .unwrap(); + + let (respond_to, _response) = oneshot::channel(); + storage + .append_executed_user_ops_chunk( + &mut head, + &[IncludedUserOp { + pending: PendingUserOp { + signed: SignedUserOp { + sender: Address::ZERO, + signature: Signature::test_signature(), + user_op: UserOp { + nonce: 0, + max_fee: u16::MAX, + data: vec![].into(), + }, + }, + respond_to, + received_at: SystemTime::now(), + }, + executed_input_offset: ExecutedInputCount::ZERO, + }], + ) + .unwrap(); + let next_id = storage.insert_pending_dump(&prefix(1), 0, 1).unwrap(); + let pending = storage + .acquire_latest_snapshot_lease(Arc::new(inline), noop_reporter()) + .unwrap() + .unwrap(); + storage.promote_finalized(0, 101).unwrap(); + storage.write(advance_recovery_generation_in).unwrap(); + + let next_version = storage.history_state().unwrap().version; + assert_eq!(next_version.era_id, first_version.era_id); + assert_eq!(next_version.recovery_generation, RecoveryGeneration::new(1)); + for leased in [&first_finalized.dump, &first_latest] { + assert_eq!(leased.prefix, prefix(0)); + assert_eq!(leased.executed_input_count, ExecutedInputCount::ZERO); + assert_eq!(leased.l2_tx_index, 0); + assert_eq!(leased.history_version, first_version); + } + assert_eq!(first_finalized.inclusion_block, 100); + assert_eq!(pending.prefix, prefix(1)); + assert_eq!(pending.executed_input_count, ExecutedInputCount::new(1)); + assert_eq!(pending.history_version, first_version); + + let next_finalized = storage + .acquire_finalized_lease(Arc::new(inline), noop_reporter()) + .unwrap() + .unwrap(); + let next_latest = storage + .acquire_latest_snapshot_lease(Arc::new(inline), noop_reporter()) + .unwrap() + .unwrap(); + for leased in [&next_finalized.dump, &next_latest] { + assert_eq!(leased.prefix, prefix(1)); + assert_eq!(leased.executed_input_count, ExecutedInputCount::new(1)); + assert_eq!(leased.l2_tx_index, 1); + assert_eq!(leased.history_version, next_version); + } + assert_eq!(next_finalized.inclusion_block, 101); + assert!(storage.gc_unreferenced_dumps().unwrap().is_empty()); + assert_eq!(storage.dump_lease_count(first_id).unwrap(), Some(2)); + assert_eq!(storage.dump_lease_count(next_id).unwrap(), Some(3)); + + drop(( + first_finalized, + first_latest, + pending, + next_finalized, + next_latest, + )); + assert_eq!(storage.dump_lease_count(first_id).unwrap(), Some(0)); + assert_eq!(storage.dump_lease_count(next_id).unwrap(), Some(0)); + assert_eq!( + storage.gc_unreferenced_dumps().unwrap(), + vec![DumpRow { + id: first_id, + prefix: prefix(0) + }] + ); + } + + #[test] + fn failed_snapshot_history_query_does_not_lease_or_arm_a_release() { + let db = temp_db("snapshot-lease-history-query-failure"); + let mut storage = Storage::open(db.path.as_str()).expect("open"); + let finalized_id = storage.insert_finalized_dump(&prefix(0), 100, 0).unwrap(); + let pending_id = storage.insert_pending_dump(&prefix(1), 0, 1).unwrap(); + storage + .conn + .execute_batch("DROP TABLE history_state") + .unwrap(); + let scheduled = Arc::new(AtomicUsize::new(0)); + + assert!( + storage + .acquire_finalized_lease(counting_scheduler(scheduled.clone()), noop_reporter()) + .is_err() + ); + assert!( + storage + .acquire_latest_snapshot_lease( + counting_scheduler(scheduled.clone()), + noop_reporter() + ) + .is_err() + ); + assert_eq!(scheduled.load(Ordering::SeqCst), 0); + assert_eq!(storage.dump_lease_count(finalized_id).unwrap(), Some(0)); + assert_eq!(storage.dump_lease_count(pending_id).unwrap(), Some(0)); + } + #[test] fn acquire_finalized_lease_reads_and_leases_atomically_blocking_gc() { let db = temp_db("acquire-finalized-lease"); From 3e5b971adf520acf9d69b2ec563825c5209df497 Mon Sep 17 00:00:00 2001 From: gcdepaula Date: Wed, 16 Sep 2026 18:06:42 -0300 Subject: [PATCH 09/29] refactor: separate application history from L1 observations Use one current application sequence with mandatory offsets and atomic recovery suffix replacement. Publish complete era baselines and derive immutable batch snapshot acceptance directly from L1 facts. Serve restorable HTTP archives and require versioned canonical WS claims. Update SDK, recovery exports, watchdog metadata, tests, and architecture documentation. --- AGENTS.md | 55 +- Cargo.lock | 4 + README.md | 75 +- docs/invariants.md | 388 ++-- docs/plans/2026-07-coordination-tracks.md | 33 +- .../2026-07-track3-feed-replay-design.md | 471 ++--- docs/plans/2026-08-authority-boundary-adr.md | 13 +- docs/plans/application-history.md | 92 + docs/protocol/application-contract.md | 8 +- docs/protocol/c-application-binding.md | 4 +- docs/protocol/scheduler-semantics.md | 10 +- docs/recovery/README.md | 28 +- docs/recovery/admission.tla | 28 +- docs/recovery/cockroach.md | 350 +--- docs/review/register.md | 48 +- docs/snapshots/format.md | 4 +- docs/snapshots/lifecycle.md | 604 ++---- docs/watchdog/README.md | 4 +- docs/watchdog/getting-started.md | 2 +- sdk/rust-client/Cargo.toml | 3 + sdk/rust-client/src/errors.rs | 10 + sdk/rust-client/src/lib.rs | 195 +- sequencer-core/src/history.rs | 74 +- sequencer/Cargo.toml | 3 +- sequencer/src/commands/config.rs | 29 +- sequencer/src/commands/error.rs | 87 +- sequencer/src/commands/run/startup_hygiene.rs | 125 +- sequencer/src/commands/run/workers.rs | 21 +- sequencer/src/commands/setup/fill.rs | 995 +++------ sequencer/src/commands/setup/mod.rs | 214 +- sequencer/src/commands/test_support.rs | 2 - sequencer/src/egress/api/snapshot.rs | 314 ++- sequencer/src/egress/api/state.rs | 3 - sequencer/src/egress/api/subscribe.rs | 96 +- sequencer/src/egress/l2_tx_feed/error.rs | 25 +- sequencer/src/egress/l2_tx_feed/mod.rs | 174 +- sequencer/src/egress/l2_tx_feed/tests.rs | 357 +--- sequencer/src/harness.rs | 11 +- sequencer/src/http.rs | 10 - .../src/ingress/inclusion_lane/catch_up.rs | 184 +- .../src/ingress/inclusion_lane/config.rs | 9 +- .../src/ingress/inclusion_lane/dump_info.rs | 226 +-- sequencer/src/ingress/inclusion_lane/error.rs | 32 +- sequencer/src/ingress/inclusion_lane/mod.rs | 152 +- .../src/ingress/inclusion_lane/snapshot.rs | 563 +----- sequencer/src/ingress/inclusion_lane/tests.rs | 491 ++--- .../integration_tests/chain_id_validation.rs | 9 +- .../src/integration_tests/e2e_sequencer.rs | 61 +- .../integration_tests/snapshot_endpoints.rs | 249 ++- .../src/integration_tests/ws_broadcaster.rs | 182 +- sequencer/src/lib.rs | 2 +- sequencer/src/recovery/mod.rs | 44 +- sequencer/src/storage/convert.rs | 2 +- sequencer/src/storage/egress.rs | 228 +-- sequencer/src/storage/egress/canonical.rs | 59 +- .../src/storage/egress/canonical/tests.rs | 83 +- sequencer/src/storage/history.rs | 385 +--- sequencer/src/storage/ingress.rs | 875 +++----- sequencer/src/storage/l1_submission.rs | 10 +- sequencer/src/storage/lifecycle.rs | 74 +- .../src/storage/migrations/0001_schema.sql | 425 +--- sequencer/src/storage/mod.rs | 17 +- sequencer/src/storage/mutations.rs | 89 +- sequencer/src/storage/open.rs | 146 +- sequencer/src/storage/queries.rs | 19 +- sequencer/src/storage/recovery.rs | 60 +- sequencer/src/storage/recovery_tests.rs | 536 +++-- .../src/storage/safe_accepted_batches.rs | 170 +- sequencer/src/storage/snapshot_dumps.rs | 1778 ++++------------- sequencer/src/storage/test_helpers.rs | 17 +- tests/benchmarks/README.md | 2 +- .../benchmarks/src/bin/round_trip_latency.rs | 10 +- tests/benchmarks/src/bin/sweep.rs | 8 - tests/benchmarks/src/round_trip.rs | 7 +- tests/e2e/src/test_cases.rs | 164 +- tests/harness/Cargo.toml | 3 + tests/harness/src/replay.rs | 8 + tests/harness/src/sequencer.rs | 148 +- tests/harness/src/ws.rs | 11 +- watchdog/sequencer_reader.lua | 10 +- watchdog/tests/drill_divergence.lua | 4 +- watchdog/tests/run.lua | 46 +- 82 files changed, 4427 insertions(+), 8140 deletions(-) create mode 100644 docs/plans/application-history.md diff --git a/AGENTS.md b/AGENTS.md index a25bd67d..22add5bb 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -188,37 +188,28 @@ Top-level layout follows the system's data flow. Each sequencer module correspon - **Danger detector** — background worker that polls `Storage::check_danger` on a fixed cadence and exits with `RecoveryRequired` when any non-`Safe` danger status fires. Never writes to the DB; never talks to L1. Crashes the process so startup recovery or refusal can run. - **Fee oracle** — setup pins either a fixed exponent or a reviewed Uniswap V3 WETH/X TWAP tuple into deployment identity, and writes the first `log_gas_price` (+ observation stamp) in both modes. Setup requires a successful live quote; `run` performs no fee-source read before recovery/admission. Fixed mode has no worker; Uniswap launches a lazy refresher that immediately attempts a quote, persists successes, and retains the last price while logging and retrying transient source failures. The stamp is telemetry, not a runtime-admission or expiry gate. A shared-endpoint outage/stale view is already detected from L1 safe-head progress; a fee-source-only outage is an accepted economic residual (stale-low may subsidize DA, stale-high may reject users), not a canonical-correctness fault. Deterministic source misconfiguration, fatal arithmetic, and persistent storage faults remain terminal. The 10× margin lives in `batch_policy.log_slack`; it is a buffer rather than a bound on market movement, and frame fees stay immutable until the next frame opens. - **Input reader** — ingests safe inputs from L1 InputBox and maintains the durable safe head, accepted-batch projection, and divergence marker in one atomic transaction (`sequencer/src/storage/l1_inputs.rs`); it hands the lane no in-memory cursor. -- **L2 tx feed** — DB-backed ordered-tx stream used by WS subscribers. The - existing endpoint still paginates by the physical SQLite rowid cursor. - SQLite now also stores the canonical `ExecutedInputCount` attribution for - every application input; switching the public feed and history-version - handshake to that coordinate remains Track 3 API work. -- **Application progress** — engine-owned, with protocol-defined semantics: - `(ExecutedInputCount, last_executed_safe_block)` embedded in every - application dump. Shared execution functions verify its transition and return the - input's pre-execution offset. SQLite records that offset atomically with the - corresponding valid replay row; only the WebSocket/HTTP projection remains - Track 3 work. -- **History version** — `(EraId, RecoveryGeneration)`. The durable metadata - foundation is landed: a new baseline mints an immutable UUIDv4 era and starts - generation zero; standard recovery increments it exactly once iff its - transaction invalidates at least one valid batch. The current feed does not - expose or enforce the pair yet. +- **L2 tx feed** — DB-backed application-input stream. HTTP snapshot headers + provide `(EraId, RecoveryGeneration, ExecutedInputCount)`; WS validates that + claim and replays inclusively before following the tip. +- **Application progress** — engine-owned `(ExecutedInputCount, + last_executed_safe_block)`, embedded in every dump. Shared execution verifies + the transition and returns the pre-execution offset; storage commits that + mandatory offset with its source in `application_inputs`. +- **History version** — `(EraId, RecoveryGeneration)`. Setup publishes a complete + baseline with a fresh era; recovery increments the generation exactly once + iff it invalidates at least one valid batch. Subscription claims enforce both. - **Soft confirmation** — sequencer's predicted ordering, emitted before the batch lands on L1. -- **Snapshot** — durable copy of the app's canonical state at one physical - replay cursor and one canonical `ExecutedInputCount`; *pending* at batch - close, *promoted* to finalized on L1 observation (per-range, atomically with - the drain), garbage-collected when superseded. Catch-up refuses if the - loaded app count, stored snapshot count, or per-row execution attributions - disagree. Lifecycle + rationale (incl. the promote/drain crash-safety): - [`docs/snapshots/lifecycle.md`](docs/snapshots/lifecycle.md). +- **Snapshot** — immutable artifact at every batch close, registered with its + local batch identity and application count. Acceptance facts select the + recovery/watchdog checkpoint; the era baseline supplies the initial restore + point. Lifecycle and leases: [`docs/snapshots/lifecycle.md`](docs/snapshots/lifecycle.md). ## Domain Truths - API validates the EIP-712 signature and enqueues a `SignedUserOp`. Method payload decoding happens during application execution, not at ingress. - **Deposits are direct-input-only** (L1 → L2) and must not be represented as user ops. - Rejections (`InvalidNonce`, `InvalidMaxFee`, `InsufficientFeeBalance`) produce no state mutation and are not persisted. These are protocol-level rejection semantics every app must implement: nonces prevent user-op replay, fees prevent spam against the sequencer's DA budget. ("Fee", not "gas" — the fee tracks DA; compute metering, if it ever exists, is a separate future concept.) -- Included txs are persisted as frame/batch data in `batches`, `frames`, `user_ops`, `safe_inputs`, and `sequenced_l2_txs`. Recovery metadata lives in `safe_accepted_batches`; batch lifecycle state (sealed/invalidated) lives on the `batches` row itself as write-once timestamps. +- Included txs are persisted as frame/batch data in `batches`, `frames`, `user_ops`, `safe_inputs`, and `application_inputs`. Recovery metadata lives in `safe_accepted_batches`; batch lifecycle state (sealed/invalidated) lives on the `batches` row itself as write-once timestamps. - Frame fee is persisted in `frames.fee` and is fixed for the lifetime of that frame. The next frame's fee is currently sampled from `batch_policy_derived.recommended_fee` at rotation; oracle bootstrap writes the price before any Tip can sample it, and `log_slack` applies the 10× margin in log space. This is present behavior, not a reason for the five-block clock policy; hoisting fee to the batch is a later design with its own trade-offs. - Wallet state (balances, nonces) is in-memory today — not persisted. - **EIP-712 domain fields:** `name`, `version`, `chainId`, `verifyingContract`. `chainId` and `verifyingContract` come from `CARTESI_SEQUENCER_BLOCKCHAIN_ID` and `CARTESI_SEQUENCER_APP_ADDRESS` (validated against the RPC chain id at startup). All four fields must be present on both sides — both the sequencer and the on-chain scheduler construct the domain via `sequencer_core::build_input_domain`, the canonical shared constructor. @@ -259,7 +250,7 @@ The hot-path rules are owned elsewhere; this section is only the map. - Drain attribution, frame-clock monotonicity, the content-identity check and the divergence freeze, history metadata, the - `WriteHead` cache, and the executed-inputs projection are registered in + `WriteHead` cache, and the application-input sequence are registered in [`docs/invariants.md`](docs/invariants.md) (the fail-loud check policy plus I2, I3, I9, I10, I12–I18, I20) — that register owns them; do not restate them here. @@ -284,8 +275,7 @@ ordering logic without explicit approval. Owned by [`docs/invariants.md`](docs/invariants.md): the writer-role table (one writer role per fact), the `valid_*` view rule, `WriteHead` coherence -(I17), history metadata (I18), the replay cursor and the executed-inputs -projection (I10, I20). The schema +(I17), history metadata (I18), the application-input sequence and its canonical offsets (I10, I20). The schema (`sequencer/src/storage/migrations/0001_schema.sql`) owns the write-once batch lifecycle, the Tip's uniqueness, and the user-op identity rule. Do not restate them here. @@ -294,18 +284,17 @@ restate them here. - `SignedUserOp` — ingress/API signature domain (post-validation, pre-execution). - `ValidUserOp` — application execution domain (after validation boundary). -- `SequencedL2Tx` — ordered replay/fanout domain (`UserOp | DirectInput`). +- `SequencedL2Tx` — application input payload sum (`UserOp | DirectInput`). - `ExecutedInputCount` — canonical application-history boundary (`X` means the next input is entry `X`), never a SQLite cursor. Checked arithmetic only. -- `ReplayL2TxRow` — crate-private named pairing of a physical DB cursor, - `SequencedL2Tx`, frame clock, and optional canonical attribution; do not - collapse these coordinates back into a positional tuple. +- `ApplicationInputRow` — crate-private pairing of a mandatory application offset + and source context. Every row executes; batch envelopes stay in `safe_inputs`. - Keep DB-only helper types private to storage modules; prefer shared domain types at module boundaries. ## HTTP Endpoints - **Ingress** (public-facing): `POST /tx`, `GET /fee`. -- **Egress** (internal indexers/watchdog): `GET /ws/subscribe`, `GET /finalized_state`, `GET /finalized_state/inclusion_block`, `GET /latest_snapshot`, `GET /livez`, `GET /readyz`, `GET /healthz`. The snapshot/state endpoints are **operator-only** (no auth) and must not be exposed publicly; the streaming routes hold a GC lease for the response lifetime ([`docs/snapshots/lifecycle.md`](docs/snapshots/lifecycle.md)). +- **Egress** (internal indexers/watchdog): `GET /ws/subscribe`, `GET /finalized_state`, `GET /finalized_state/inclusion_block`, `GET /latest_snapshot`, `GET /finalized_snapshot`, `GET /livez`, `GET /readyz`, `GET /healthz`. The snapshot/state endpoints are **operator-only** (no auth) and must not be exposed publicly; the streaming routes hold a GC lease for the response lifetime ([`docs/snapshots/lifecycle.md`](docs/snapshots/lifecycle.md)). Today both sides serve from one listener; the planned API split puts each side on its own port (same binary) so internal probes and subscribers can be firewalled from public submit traffic. @@ -465,7 +454,7 @@ Before finishing a change, ensure: - [`docs/plans/`](docs/plans/) — the architecture decision record ([`2026-08-authority-boundary-adr.md`](docs/plans/2026-08-authority-boundary-adr.md)), active coordination tracks, and in-flight design handoffs. - [`docs/threat-model/README.md`](docs/threat-model/README.md) — trust boundaries, in-scope and out-of-scope threats. - [`docs/recovery/README.md`](docs/recovery/README.md) — recovery design, TLA+ formal verification, design history. -- [`docs/snapshots/`](docs/snapshots/) — app snapshots: [`format.md`](docs/snapshots/format.md) (dump trait + wire format) and [`lifecycle.md`](docs/snapshots/lifecycle.md) (take/promote/GC/lease design + crash-safety). +- [`docs/snapshots/`](docs/snapshots/) — app snapshots: [`format.md`](docs/snapshots/format.md) (dump trait + wire format) and [`lifecycle.md`](docs/snapshots/lifecycle.md) (creation/acceptance/GC/lease design + crash-safety). - [`docs/watchdog/operator-deployment.md`](docs/watchdog/operator-deployment.md) — production-like watchdog (Sepolia / mainnet; internal snapshot API). - [`docs/watchdog/getting-started.md`](docs/watchdog/getting-started.md) — local dev: watchdog + `sequencer-devnet` on Anvil. - [`docs/watchdog/README.md`](docs/watchdog/README.md) — watchdog architecture, compare vs advance modes, test commands. diff --git a/Cargo.lock b/Cargo.lock index 18a0ed38..81d393d9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3899,14 +3899,17 @@ dependencies = [ "ethereum_ssz", "futures-util", "k256", + "reqwest 0.12.28", "rusqlite", "sequencer-core", "sequencer-rust-client", "serde", "serde_json", + "tar", "tempfile", "tokio", "tokio-tungstenite 0.28.0", + "toml", ] [[package]] @@ -4287,6 +4290,7 @@ dependencies = [ "sequencer-rust-client", "serde", "serde_json", + "tar", "tempfile", "thiserror 2.0.19", "tokio", diff --git a/README.md b/README.md index 01a0ec35..fe414f9f 100644 --- a/README.md +++ b/README.md @@ -81,7 +81,7 @@ Users submit signed operations via `POST /tx` (JSON). Operations are signed with ### Sequenced Transaction Feed -Subscribers connect via `GET /ws/subscribe?from_offset=` (WebSocket). The feed delivers all sequenced transactions (user ops + direct inputs) in deterministic order, matching the on-chain execution order. This is the primary interface for downstream consumers (frontends, indexers). The endpoint is designed for a small number of indexer subscribers, which serve users directly. +Subscribers connect via `GET /ws/subscribe?era_id=&recovery_generation=&next_input=` (WebSocket). The feed delivers all sequenced transactions (user ops + direct inputs) in deterministic order, matching the on-chain execution order. This is the primary interface for downstream consumers (frontends, indexers). The endpoint is designed for a small number of indexer subscribers, which serve users directly. ### Batch Submission @@ -184,18 +184,27 @@ Notes: - `200` while an open frame exists (the admitted runtime always has one). - `503` with code `UNAVAILABLE` during shutdown, or if no open frame exists. -### `GET /ws/subscribe?from_offset=` - -WebSocket stream of sequenced L2 transactions from persisted order. - -Notes: - -- `from_offset` is optional and defaults to `0`. -- messages are JSON text frames. -- binary fields are hex-encoded (`0x`-prefixed). -- direct-input `block_timestamp` values are Unix seconds. -- the current runtime enforces a subscriber cap of `64` and a catch-up cap of `50000` events. -- if the requested catch-up window exceeds that cap, the server upgrades and then immediately closes the socket with close code `1008` (`POLICY`) and reason `catch-up window exceeded: live_start_offset=`; reconnecting at that offset starts from the current live head. +### `GET /ws/subscribe?era_id=&recovery_generation=&next_input=` + +WebSocket stream of canonical application inputs, replaying from the inclusive +`next_input` offset and then following the optimistic tip. Fetch and restore +`/latest_snapshot` first; its headers supply the complete subscription claim. +After each successfully applied input at offset `X`, persist the claim with +`next_input = X + 1` alongside the replica state. + +- All three query fields are required. Missing or malformed fields return HTTP `400`. +- An era or generation mismatch, an unavailable prefix, or a position ahead of + the head returns HTTP `409` before upgrade. The JSON body and `X-History-Error` + header carry the same typed refusal: `ERA_CHANGED`, `STALE_GENERATION`, + `HISTORY_UNAVAILABLE`, or `AHEAD_OF_HEAD`. Rebootstrap on a history mismatch. +- A claim exactly at the head waits for the next input. Replay uses bounded + pages and queues, with no total catch-up limit. The subscriber cap is `64`. +- Messages are JSON text frames; binary fields are `0x`-prefixed hex. + Direct-input `block_timestamp` values are Unix seconds. +- Batch envelopes are absent. Offsets count executed application inputs, + including business failures and malformed-direct no-ops. +- Recovery stops the process and disconnects subscribers. A reconnect must + present its saved claim; offsets alone cannot distinguish a replaced suffix. Message shapes: @@ -224,18 +233,26 @@ These serve application state to the operator's watchdog and indexers. (gated by network controls today; bound to a separate internal port once the api split lands). -- `GET /finalized_state/inclusion_block` — cheap JSON the watchdog polls to - detect advance: `{ "inclusion_block": , "l2_tx_index": }`. `404` - if no finalized snapshot exists. -- `GET /finalized_state` — streams the L1-finalized state file - (`application/octet-stream`); headers `X-Inclusion-Block`, `X-L2-Tx-Index`, - and `ETag: "block-"` (send `If-None-Match` for a `304`). -- `GET /latest_snapshot` — streams the latest snapshot (latest pending if any, - else finalized) for indexers that fetch state then subscribe at - `X-L2-Tx-Index`. - -Both streaming routes hold a GC lease on the dump for the response lifetime, -released even on client disconnect. +- `GET /finalized_state/inclusion_block` — cheap JSON the watchdog polls: + `{ "inclusion_block": , "executed_input_count": }`. +- `GET /finalized_state` — streams the accepted checkpoint's comparison file + (`application/octet-stream`), with `X-Inclusion-Block`, + `X-Executed-Input-Count`, and `ETag: "block-"` (`If-None-Match` supports `304`). + The watchdog compares at the end of that L1 block. +- `GET /latest_snapshot` — streams a restorable tar archive of the newest valid + batch-close snapshot, or the era baseline. Includes immutable `info.toml` + and the application's opaque `state` file or directory. +- `GET /finalized_snapshot` — streams the accepted snapshot as a tar archive, + adding a coherent `checkpoint.toml` receipt with its L1 inclusion block and + next batch nonce for trusted recovery. + +All state/archive responses include `X-History-Era`, `X-Recovery-Generation`, +and `X-Executed-Input-Count`, selected atomically with the artifact lease. +Streaming holds the lease until the response ends or the client disconnects. +The accepted endpoints return `404` until a comparable checkpoint exists: +genesis is comparable at block zero; a rebuilt baseline is restorable but only +a later accepted batch establishes a comparison point. Divergence blocks +publication of the accepted checkpoint. See [snapshot lifecycle](docs/snapshots/lifecycle.md). ## Storage Model @@ -243,8 +260,10 @@ released even on client disconnect. - `frames`: frame boundaries within each batch - `frames.fee`: committed fee for each frame - `user_ops`: included user operations -- `sequenced_l2_txs`: append-only ordered replay rows (`UserOp` xor `DirectInput`); inserting into `user_ops` also appends the corresponding replay row via trigger `trg_sequence_user_op` -- `safe_inputs`: direct-input payload stream +- `application_inputs`: current application sequence keyed by mandatory pre-execution offset; each row references a user op or an external direct input and its owning batch/frame +- `safe_inputs`: every raw InputBox observation, including batch envelopes +- `history_state`: immutable era baseline (application count and accounted L1 block) plus recovery generation +- `snapshots` and `dumps`: immutable batch-close/baseline artifacts and streaming leases; accepted status is derived from `safe_accepted_batches` - `batch_policy`: singleton knobs and constants for DA-style batch sizing and fee derivation; `batch_policy_derived` exposes `recommended_fee` and `batch_size_target`. A batch closes on whichever fires first: the derived `batch_size_target` byte budget or the `max_batch_open` wall-clock deadline (an inclusion-lane setting, `CARTESI_SEQUENCER_MAX_BATCH_OPEN_SECONDS`, not a `batch_policy` column). Setup writes the first `log_gas_price` (and observation stamp) for both Fixed and Uniswap modes, failing if the initial Uniswap quote cannot be read. Fixed local pricing has no oracle worker; Uniswap starts from the persisted price and refreshes lazily via the setup-pinned WETH/fee-token TWAP source, retaining that price across transient source failures. `log_slack = log(10)` applies the 10× safety margin in log space. Fees are app-token smallest units — initially USDC (6 decimals) for the wallet prototype — not a protocol-level USDC invariant. ## Project Layout @@ -281,7 +300,7 @@ docker pull ghcr.io/cartesi/sequencer-watchdog:vX ## Prototype Limits -- The `Application` trait exposes snapshot dump/load capability (format in `docs/snapshots/format.md`). The inclusion lane drives the snapshot lifecycle — dump at batch close, promote to finalized on L1 observation, and garbage-collect superseded dumps — and at startup rebuilds application state by loading the latest snapshot and replaying the persisted L2-tx stream from that snapshot's offset. The lifecycle and its rationale (per-range atomic promotion, GC, leasing, crash-safety) are documented in `docs/snapshots/lifecycle.md`. The snapshot is served to the operator's watchdog/indexers over internal-only HTTP routes (`/finalized_state`, `/finalized_state/inclusion_block`, `/latest_snapshot`) — no auth, gated by network-level access control until the planned per-port api split lands. +- The `Application` trait defines dump/load behavior ([format](docs/snapshots/format.md)). Every batch close registers a durable snapshot atomically with the seal. Restart restores the latest valid snapshot and replays application inputs from its count. Acceptance determines the recovery checkpoint and garbage-collection frontier without mutating artifact metadata. [Snapshot lifecycle](docs/snapshots/lifecycle.md) documents leases and crash ordering. - Schema and migrations are still in prototype mode and may change. ## Local Test Prerequisites diff --git a/docs/invariants.md b/docs/invariants.md index cf56da55..802fe888 100644 --- a/docs/invariants.md +++ b/docs/invariants.md @@ -72,21 +72,20 @@ don't. ### Writer roles One writer role per fact. Reads over batch data go through the `valid_*` -views (`valid_batches`, `valid_closed_batches`, `valid_open_batch`, -`valid_sequenced_l2_txs`), which encapsulate the "exclude invalidated rows" +views (`valid_batches`, `valid_closed_batches`, `valid_open_batch`), which encapsulate the "exclude invalidated rows" filter; writers target the base tables. The batch lifecycle columns partition by writer and are write-once (`0001_schema.sql`). | Writer | Writes | |---|---| -| inclusion lane | `batches` (insert + `sealed_at_ms`), `frames`, `user_ops`, `sequenced_l2_txs`, `executed_inputs`, `dumps`/`pending_snapshots` (batch close), `finalized_snapshot` (promotion only — setup registers the initial row) | +| inclusion lane | `batches` (insert + `sealed_at_ms`), `frames`, `user_ops`, `application_inputs`, `dumps`/`snapshots` (batch close) | | input reader | `safe_inputs`, `l1_safe_head`, `safe_accepted_batches`, `canonical_divergence` (the divergence poison marker) | -| recovery (startup) | `batches.invalidated_at_ms`, Tip reopen, scoped `pending_snapshots` clear, derived `executed_inputs` suffix deletion | -| history metadata (setup/recovery) | `history_state` — era/generation at baseline, generation bump in a non-empty standard-recovery cascade, rebuild application base + safe-input drain floor at initial finalized-snapshot registration | +| recovery (startup) | `batches.invalidated_at_ms`, Tip reopen, current `application_inputs` suffix deletion | +| history metadata (setup/recovery) | `history_state` — complete era/application-count/L1-block baseline, generation bump in a non-empty standard-recovery cascade | | batch submitter and mempool flusher | `wallet_nonce_watermark` — deliberately shared under one protocol: each raises it before its first broadcast (write-before-broadcast, I14) | | egress (HTTP) | `dumps.lease_count` (leases); `run`'s startup hygiene resets it to zero as the crash backstop | -| setup | `deployment_identity` (pinned once), `batch_tree_anchor` (the root nonce, frozen once setup completes), the initial `dumps` + `finalized_snapshot` rows (genesis or rebuild registration, atomic with the history bases), the `setup_complete` fact (written once), `batch_policy.log_gas_price` + `log_gas_price_updated_at_ms` (first write; Fixed and Uniswap) | -| snapshot GC (the lane after a promotion, `run`'s startup hygiene) | unreferenced `dumps` row deletion (`gc_unreferenced_dumps`) | +| setup | `deployment_identity` (pinned once), `batch_tree_anchor` (the root nonce, frozen once setup completes), the initial `dumps` + `snapshots` rows (genesis or rebuild registration, atomic with the complete history baseline), the `setup_complete` fact (written once), `batch_policy.log_gas_price` + `log_gas_price_updated_at_ms` (first write; Fixed and Uniswap) | +| snapshot GC (the lane after reconciliation, `run`'s startup hygiene) | unreferenced `dumps` row deletion (`gc_unreferenced_dumps`) | | command brackets (run, setup, flush) | `terminal_faults` (append-only, best-effort at settlement) | | admin | `batch_policy` alpha knobs (`log_alpha`, `log_one_plus_alpha`) | | fee oracle | `batch_policy.log_gas_price` + `log_gas_price_updated_at_ms` (Uniswap mode only; stamps on every successful refresh) | @@ -106,7 +105,7 @@ by writer and are write-once (`0001_schema.sql`). the canonical fold and the predicate diverge exactly and only there.) - **Enforced by:** review + the duality test. No structural mechanism. - **Depended on by:** everything — the gold frontier, recovery's cascade pivot, - promotion, soft-confirmation honesty. + checkpoint selection, soft-confirmation honesty. - **Breaks:** silent permanent scheduler/sequencer divergence. - The expected-nonce fold is homed next to `scheduler_accepts` as `advance_expected_batch_nonce`; `decide_submit_start` consumes it, while @@ -123,7 +122,7 @@ by writer and are write-once (`0001_schema.sql`). several below-threshold observations. Frame K's wire content is therefore "directs ≤ S_K, then ops validated on top"; a clock tick with no directs is an empty-prefix instance of the same rule. That leading direct prefix is - recoverable from `sequenced_l2_txs` plus `frames.safe_block` alone. + recoverable from `application_inputs` plus `frames.safe_block` alone. - **Enforced by:** `close_frame_in` ordering; lane convention. - **Depended on by:** the duality (scheduler's drain-before-ops equals the flattened replay order); catch-up; the feed. @@ -150,75 +149,57 @@ by writer and are write-once (`0001_schema.sql`). within-batch monotonicity check; "if the frontier batch is fresh, all are". - **Breaks:** I4's guarantee evaporates; danger detection mis-orders. -### I4. Tip-only cascade ⇒ every closed batch is gold - -- **Holds:** `check_danger` checks `ClosedBatchInDanger` **before** - `TipInDanger`; with I3, the closed frontier is always at least as old as the - Tip, so the Tip arm can only fire when no non-gold closed batch exists. -- **Enforced by:** the arm order in `Storage::check_danger` - (`storage/recovery.rs`) + I3. -- **Depended on by:** the dispatch table's meaning (a `RecoverTip` boot may - skip the flush *because* nothing closed is doomed). **Not load-bearing for - the pending clear**: the clear is scoped to `nonce >= pivot.nonce` in - `cascade_and_reopen`, so a valid in-flight closed batch's pending survives - any cascade by construction, regardless of arm order. -- **Breaks:** a Tip-only cascade while a closed batch is doomed would leave - the doomed batch un-cascaded until the next detector cycle (liveness lag, - not the old crash-loop). - -### I5. Pending-clear is scoped to the cascade and runs in its transaction - -- **Holds:** recovery deletes only pending rows with `nonce >= - pivot.nonce`, atomically with the cascade and the full-backlog tip reopen - (`cascade_and_reopen`, `storage/recovery.rs`). -- **Enforced by:** the single `write` tx + the scoped - `clear_pending_dumps_from_nonce_in`. -- **Depended on by:** catch-up never loading a cascaded batch's state - (cleared rows), and promotion never hitting a deleted row for a batch that - stayed valid (surviving rows) — the `lifecycle.md` §6/§8 wedge is - unrepresentable. -- **Breaks:** widening the delete re-arms the promote-wedge crash-loop; - narrowing it lets catch-up resume from a cascaded batch's state. - -### I6. A committed promotion implies an advanced drain - -- **Holds:** promotion is folded into the drain's transaction - (`close_frame_only_with_executions`), together with canonical - direct-input attribution. -- **Enforced by:** the single `write` tx in `storage/ingress.rs`; the - standalone `Storage::promote_finalized` is `#[cfg(test)] pub(crate)`. -- **Depended on by:** crash-safety of the safe-frontier walk - (`lifecycle.md` §5–§6). -- **Breaks:** restart re-processes the range and re-promotes a deleted pending - row — crash-loop. - -### I7. A committed batch close has a promotable pending row - -- **Holds:** seal + next-Tip open + `pending_snapshots` insert commit together - (`close_frame_and_batch_with_pending_dump`). -- **Enforced by:** single transaction; `create_dump` happens before, on disk. -- **Depended on by:** promotion (`promote_finalized_in` hard-fails on a missing - row). -- **Breaks:** promotion wedge at the sealed batch's landing. - -### I8. Always-load: a finalized snapshot and a valid Tip exist before the lane starts - -- **Holds:** cold start registers the genesis dump as finalized and opens the - genesis Tip; recovery reopens the Tip atomically across cascades. -- **Enforced by:** `setup` atomically registers the genesis finalized snapshot - before its completion fact; startup recovery refuses a missing finalized - fact, opens a missing Tip only through `ensure_open_tip_for_recovery`, - and recovery's cascade reopens in-transaction. `PreparedRuntime::prepare` - reasserts the snapshot artifact before admission. -- **Depended on by:** catch-up's unconditional load path - (`CatchUpError::NoSnapshot` is fail-loud, not a branch); the lane's - `NoOpenTip` fail-loud load. -- **Breaks:** startup crash (loud — by design). +### I4. Closed-frontier danger takes precedence over Tip danger + +- **Holds:** `check_danger` checks `ClosedBatchInDanger` before `TipInDanger`. + With monotonic frame clocks, the closed frontier is at least as old as the Tip. +- **Enforced by:** arm order in `storage/recovery.rs` and I3. +- **Depended on by:** dispatch: a Tip-only recovery can skip flushing because + there is no doomed closed work. + +### I5. Recovery removes exactly the invalidated application suffix + +- **Holds:** invalidating a batch deletes its `application_inputs` through the + schema trigger. The cascade, generation increment, and replacement Tip commit + together. Original source records and immutable snapshots remain; snapshot + selection excludes invalidated batches and GC retires their unleased artifacts. +- **Enforced by:** `cascade_and_reopen`, application-input constraints, valid views. +- **Breaks:** loading invalidated state or leaving a hole in current history. + +### I6. The frame clock accounts for a complete L1 interval + +- **Holds:** the surviving latest frame clock, floored by baseline block `C`, + identifies the completely accounted L1 prefix. Reconciliation executes all + external directs in the newly safe interval before committing its next frame + and application rows. Envelopes-only intervals advance the clock with no rows. +- **Enforced by:** complete-block ingestion; the lane's indivisible reconciliation + turn; storage range and execution-receipt checks. Batch closure preserves the + current frame clock. +- **Breaks:** skipping or double-applying directs after restart/recovery. + +### I7. Every committed batch close has an immutable snapshot + +- **Holds:** file creation precedes the transaction sealing the batch, opening + the next Tip, and registering its snapshot by local `batch_index` and count. +- **Enforced by:** `close_batch_with_snapshot` and + `close_frame_and_batch_with_snapshot`. Selection requires the exact expected + batch's snapshot; a missing artifact fails loud. +- **Breaks:** losing the accepted recovery/watchdog checkpoint. + +### I8. A rollback-safe snapshot and valid Tip exist before the lane starts + +- **Holds:** setup publishes the complete durable baseline before completion; + recovery retains the newest accepted snapshot, or that baseline until first + acceptance. A rebuilt baseline is a restore point, not a comparison at `C`. +- **Enforced by:** `complete_baseline_setup`, recovery admission, atomic Tip + reopen, and `PreparedRuntime::prepare` artifact checks. +- **Depended on by:** unconditional snapshot restore and catch-up. +- **Breaks:** startup fails loud instead of inventing state. ### I9. Acceptance identity: "accepted nonce N" means "our valid batch N" - **Holds:** by nonce **and content** — the **content-identity check**: every - landing at/above the batch-tree anchor that the off-chain + landing strictly after baseline block `C` that the off-chain `scheduler_accepts` simulation accepts is compared against the local valid closed batch at that nonce — `keccak256(landed bytes)` vs the hash stamped at seal by the same encode path the submitter broadcasts. The exhaustive local @@ -235,7 +216,7 @@ by writer and are write-once (`0001_schema.sql`). detection — the content-identity check in `populate_safe_accepted_batches`, which on violation persists the `canonical_divergence` marker and freezes the frontier (I15). -- **Depended on by:** the gold frontier, cascade pivot selection, promotion, +- **Depended on by:** the gold frontier, cascade pivot selection, checkpoint selection, local-state ↔ canonical-state agreement. - **Breaks:** would be silent divergence (a zombie replay of our own stale tx winning a nonce slot; a power-loss re-seal at the same nonce with different @@ -258,32 +239,24 @@ by writer and are write-once (`0001_schema.sql`). soft confirmations issued inside that window are built on already-diverged state — bounded, and those confirmations are rollbackable by design. -### I10. Replay-offset sentinel: `0` means "from genesis" - -- **Holds:** `valid_ordered_l2_tx_head` returns 0 on an empty stream, and - catch-up pages with `offset > cursor` — sound because `sequenced_l2_txs` - rowids start at 1 and rows are never deleted (invalidated rows are filtered, - not removed), so offsets are globally increasing and 0 is never a real - offset. -- **Enforced by:** SQLite rowid semantics + append-only convention. -- **Depended on by:** catch-up, the feed cursor, snapshot `l2_tx_index`. -- **Breaks:** first transaction skipped or double-applied on replay. -- **Scope:** this is the current physical SQLite replay cursor. It is not the - canonical `Application::executed_input_count()` feed coordinate. The - canonical mapping is durable, but the public feed has not changed from rowid - pagination yet. - -### I11. Own-batch safe inputs are sequenced but never executed or fanned out - -- **Holds:** batch-submitter-sent safe inputs enter `sequenced_l2_txs` like any - drained input, but are skipped by sender at catch-up replay - (`catch_up.rs`), at live execution (`execute_safe_inputs_chunk`), and at WS - delivery (feed filter). -- **Enforced by:** sender checks at each consumer (three places — keep them in - sync). -- **Depended on by:** replay correctness (a batch payload must never execute as - a deposit); feed consumers' state. -- **Breaks:** batch bytes applied as a direct input — divergence. +### I10. Replay uses an inclusive application-history boundary + +- **Holds:** `ExecutedInputCount = X` means input `X` executes next. Current + rows cover `[K, H)`; snapshots at `X` replay with `offset >= X`. `H` waits. +- **Enforced by:** integer primary key, contiguous insertion, coherent versioned + pages, and pre-execution catch-up count checks. +- **Depended on by:** restart and replica resume without skipping or duplication. + +### I11. Batch envelopes remain outside application history + +- **Holds:** `safe_inputs` retains every InputBox observation. The storage/lane + boundary selects external directs by the setup-pinned submitter address; + only those inputs and included user ops enter `application_inputs`. +- **Enforced by:** classified direct reads and complete receipt validation at + append. Startup/recovery derive the initial direct rows before catch-up, + which must execute them successfully before admission. Replay and WS need + no envelope filter because every row executes. +- **Depended on by:** application replay and replicated state correctness. ### I12. Safe head advances only on real observation; `synced_at_ms` is genuine progress time @@ -335,11 +308,10 @@ by writer and are write-once (`0001_schema.sql`). fails the content-identity check writes the `canonical_divergence` singleton **in the same transaction** as the sync that detected it, and `populate_safe_accepted_batches` returns early whenever the marker exists — - so no acceptance row, no promotion, and no gold-frontier advance can ever + so no acceptance row or gold-frontier advance can ever happen past a detected divergence. - **Enforced by:** the `trg_*_frozen_on_divergence` trigger family - (`0001_schema.sql`) — specifically batch-tree writes, promotions, and - pending-snapshot clears RAISE in the engine while the marker exists. This is + (`0001_schema.sql`) — specifically batch-tree writes and snapshot collection RAISE in the engine while the marker exists. This is the immediate persisted freeze for those named tables, not a general user-op hot-path barrier. The accepted frontier itself has no trigger: its single writer refuses past the marker — the guard at the top of @@ -358,8 +330,7 @@ by writer and are write-once (`0001_schema.sql`). reading `check_danger` on its poll interval (`DANGER_DETECTOR_POLL_INTERVAL`). Independently, the inclusion lane's existing time-gated SQLite read returns `SafeFrontierState::CanonicalDivergence` instead of an `Open` frontier when the marker is already present. The lane then exits - with a terminal error, causing the supervisor to abort, before direct execution, - promotion, or the five-block rotation decision. This is opportunistic + with a terminal error, causing the supervisor to abort, before direct execution or the five-block rotation decision. This is opportunistic refusal at an existing read, not another detector or a timing guarantee. One bounded dequeue chunk (`max_user_ops_per_chunk`) is the fast-turn limit, so rejected traffic cannot starve the read once its time gate is due. There is deliberately no @@ -367,32 +338,26 @@ by writer and are write-once (`0001_schema.sql`). - **Race bound:** a lane turn that already read `Open` may finish if the reader commits divergence concurrently. Preventing that would require a lock or transaction spanning application execution. Existing freeze triggers stop - conflicting batch-tree/promotion writes; the detector and next typed read + conflicting batch-tree writes; the detector and next typed read stop the process. A chunk committed before either runtime observation may acknowledge and later roll back. -- **Watchdog boundary:** the freeze stops finalized promotion before the +- **Watchdog boundary:** the freeze blocks accepted-checkpoint publication before the offending landing becomes a comparable sequencer checkpoint. Because the watchdog skips replay when the finalized inclusion block is unchanged, it does not subsume this wire-identity detector. Conversely, the check does not subsume the watchdog's broader independent application-state comparison. - **Depended on by:** standard recovery never running on a diverged frontier - (a flush+cascade there would compound the divergence); the lane never - promoting a diverged landing; the remedy being cockroach recovery only. + (a flush+cascade there would compound the divergence); egress never + publishing a diverged landing; the remedy being cockroach recovery only. - **Breaks:** silent permanent scheduler/sequencer divergence — the theft-equivalent failure. -- **Anchor-aware frontier:** the content-identity check fires - only at/above the batch-tree **anchor** ([I16](#i16-the-batch-tree-has-exactly-one-valid-parentless-root-carrying-the-deployments-anchor-nonce)). - `populate_safe_accepted_batches` seeds its initial expected nonce from the - anchor (0 for genesis — unchanged; `N'` for a cockroach-recovered deployment), - so L1 landings *below* `N'` are skipped by nonce-mismatch — they are **trusted - collapsed history**, folded into the recovered checkpoint `S'`, not foreign. - This only affects the empty-frontier seed; a running sequencer (non-empty, - append-only `safe_accepted_batches`) always resumes from `latest_accepted`, so - its foreign/zombie detection is byte-identical. `setup --recovery` itself - *defers* frontier population entirely (`InputReader::set_frontier_mode(DeferUntilAnchorSet)`): - its syncs run against an empty tree, so a frontier built then would falsely - diverge — `run`'s first sync populates it once the anchor is set. +- **Baseline-aware frontier:** accepted-batch scanning starts strictly after + the immutable baseline L1 block `C`, seeded with anchor nonce `N'`. The whole + prefix is opaque, including previously rejected future-nonce batches; it must + never be reinterpreted using a later expected nonce. Subsequent scans resume + after the last acceptance. Rebuild defers the projection until the complete + baseline and anchor are published. ### I16. The batch tree has exactly one valid parentless root, carrying the deployment's anchor nonce @@ -400,8 +365,9 @@ by writer and are write-once (`0001_schema.sql`). `parent.nonce + 1`, except the single parentless root, which carries the `batch_tree_anchor` nonce — `0` for a genesis deployment, `N'` for a cockroach-recovered one (`setup --recovery` writes the anchor before the - `setup_complete` marker). `run`'s first tip *is* that root (there is no - separate sentinel batch). A fully-torn cascade re-roots parentless at the + `setup_complete` marker). The first Tip is that root (there is no separate sentinel batch): plain + setup leaves its creation to startup recovery; rebuild creates it at `C` in + the baseline transaction. A fully-torn cascade re-roots parentless at the same anchor via `open_fresh_tip_in_tx`'s `parent = None` path, after invalidating the old root — so only one *valid* parentless root ever exists, invalidated ones coexisting. @@ -415,8 +381,9 @@ by writer and are write-once (`0001_schema.sql`). `valid_closed_batches` with `nonce >= frontier_nonce`, where `frontier_nonce` defaults to the anchor (`= N'`) while `safe_accepted_batches` is still empty after recovery, so the submitter starts at `N'` rather than 0; the recovery - fill roots the rebuilt tree at `N'` without replaying history. (`N'` is trusted - checkpoint metadata, not re-verified at setup — see + fill roots the rebuilt tree at `N'` without replaying history. (`N'` is fold-derived from trusted + checkpoint nonce `N`; wrong-low and wrong-high `N` are outside the supported + checkpoint model — see [`docs/recovery/cockroach.md`](recovery/cockroach.md#data-dictionary).) - **Breaks:** a tree mis-anchored at the wrong nonce ⇒ `run`'s first batch carries a nonce the scheduler rejects ⇒ the sequencer is wedged (never @@ -436,9 +403,8 @@ by writer and are write-once (`0001_schema.sql`). the `Storage::append_executed_user_ops_chunk`/attributed `close_*` update ordering; and the Tip, frame-position, FK, and PK constraints that fail loud on dangerous stale - cache writes. Direct-input uniqueness still depends on the lane's drain - cursor discipline because invalidated-history re-drain forbids a global - `safe_input_index` uniqueness constraint. + cache writes. Direct-input uniqueness is enforced in the current application sequence; + invalidation removes the old row before recovery can reuse its source. - **Depended on by:** the hot path avoiding a redundant SQLite re-read on every chunk; batch-size/frame counters; safe-block drain attribution; every storage method that trusts the passed head. @@ -450,56 +416,22 @@ by writer and are write-once (`0001_schema.sql`). simplify the lane, but is an independent benchmarked change rather than part of the lane-reconciliation cutover. -### I18. History metadata changes atomically with the history fact it describes - -- **Holds:** an authority-bearing initial setup/rebuild baseline creates the - schema, one immutable UUIDv4 `EraId`, and `RecoveryGeneration = 0` in one - `synchronous=FULL` transaction. Plain - setup starts with both bases zero; rebuild starts with - `base_executed_input_count = NULL` and `base_safe_input_index = NULL` - because neither the folded application nor its recovery-root cursor exists - yet. -- **Standard recovery:** `cascade_and_reopen` advances the generation exactly - once in its transaction iff it invalidates at least one valid batch. A - missing-Tip ensure or any other no-invalidation path leaves it unchanged. -- **Cockroach bind:** fill derives `K` from - `S'.executed_input_count()` and captures the recovery root's exclusive - safe-input cursor after sequencing its `<= C` padding. It binds both values - in the same transaction that registers the initial finalized snapshot. - `complete_setup` refuses while either base remains NULL or the finalized - snapshot is absent. The pair is write-once. On retry, a matching root Tip - plus that atomically bound snapshot/base pair is authoritative; it is not - re-compared with a later fold. -- **Durable drain floor:** the next-undrained cursor is the maximum of - `base_safe_input_index` and `MAX(valid safe_input_index) + 1`. Standard - recovery may invalidate the cockroach root and thereby remove its padding - from the valid view, but can never make inputs already represented by `S'` - drainable or executable again. NULL is interpreted as zero only while - setup has not completed (only a pre-completion rebuild fill can present a - NULL floor: plain setup binds base 0 in its baseline transaction, and - completion refuses while the base is NULL). -- **Coordinate separation:** `K` is an application-history boundary. It is - deliberately independent of snapshot `l2_tx_index` and the current rowid - feed cursor, which may include sequenced-but-not-executed cursor-padding - rows. The per-input projection is now durable, but the current public feed - still uses the physical cursor; its API/WS projection remains deferred. -- **Enforced by:** `baseline_migration` (`storage/open.rs`), the immutable-era, - write-once-base, and exact-`+1` schema triggers; `cascade_and_reopen` - (`storage/recovery.rs`); `insert_initial_finalized_dump` - (`storage/snapshot_dumps.rs`); and `complete_setup` - (`storage/lifecycle.rs`). -- **Depended on by:** standard-recovery discontinuity detection, honest - post-cockroach history availability, the canonical offset projection, and - the future Track 3 history-version/API protocol. -- **Breaks:** a client can mistake a rolled-back soft suffix for unchanged - history, or a rebuilt deployment can advertise an unavailable/incorrect - numeric prefix. Either silently diverges a mirror. -- **Operational boundary:** cockroach recovery remains an explicit - fresh/wiped-directory operator action. Retaining an early incomplete DB - reuses its still-unexposed era; a fail-loud partial-fill refusal requires a - wipe/retry and therefore a new unexposed era. No automated replacement, - clone detection, distributed fencing, or general resume state machine is - implied. +### I18. History identity is published with the complete baseline + +- **Holds:** file-first setup publishes `(EraId, generation=0, K, C, N')`, the + baseline snapshot, any recovery root, and setup completion in one FULL + transaction. The history row is absent before this boundary. `K` and `C` + remain immutable even after baseline artifact GC or recovery-root invalidation. +- **Standard recovery:** one generation increment iff a valid batch is + invalidated, in the cascade transaction. Clean restart changes neither token. +- **Enforced by:** `complete_baseline_setup`, immutable history triggers, + exact-`+1` generation trigger, and `cascade_and_reopen`. +- **Depended on by:** mandatory snapshot-derived WS claims. Identity is validated + before the requested count, including for empty history. +- **Breaks:** a client silently resumes a replaced suffix or inaccessible prefix. +- **Operational boundary:** rebuilding uses a fresh/wiped data directory. + Checkpoint state, inclusion block, and next nonce are trusted operator inputs; + neither clone detection nor distributed fencing is implied. ### Do-not-simplify (deliberate shapes that look like cleanup targets) @@ -511,18 +443,12 @@ like a simplification and would break a registered invariant: - **Don't reorder `check_danger`'s arms** or merge its two `find_*` helpers into one that consults the Tip first — the closed-frontier-first order is the dispatch table's meaning (I4). -- **Don't "deduplicate" promotion out of the drain transaction** — a - standalone promotion re-opens the promote-wedge crash loop (I6). -- **Don't filter own-batch rows out of `valid_sequenced_l2_txs`** — the - drain cursor is `MAX(safe_input_index)+1` over those very rows; a view - filter would rewind it and re-drain. Sender filtering stays at the - consumers (I11). -- **Don't replace the rowid offset with count-based pagination** — - invalidated-batch holes and the 0-sentinel depend on current physical - behavior (I10). -- **Don't move snapshot GC off the promotion path** to an idle loop or a - dedicated worker — promotion-coupled GC is starvation-proof and - single-writer by design (`docs/snapshots/lifecycle.md`). +- **Don't derive application order from L1 positions** — optimistic user ops + precede their envelope and have no general one-to-one L1 mapping. +- **Don't retain a numeric resume offset without its history identity** — + recovery deliberately reuses suffix counts (I18). +- **Don't discard a valid snapshot beyond the accepted frontier** — that exact + batch can become the next required recovery checkpoint (I7). - **Don't add internal retry loops to the flusher/submitter for provider errors** — the orchestrator respawn is the retry mechanism; internal retries mask exactly the failures the danger machinery routes on. @@ -545,7 +471,7 @@ like a simplification and would break a registered invariant: self-trusted; progress ownership does not require a Rust-side mirror. - **Depended on by:** the canonical scheduler, inclusion lane, catch-up, recovery fold, cockroach base `K`, durable execution attribution, and the - future Track 3 API projection. + versioned replica protocol. - **Breaks:** an input can be applied without advancing history, an offset can advance twice, or recovery can derive the wrong checkpoint clock — silent application-history divergence. @@ -553,57 +479,27 @@ like a simplification and would break a registered invariant: A failing hook is not rolled back; every production caller terminates that path and discards the instance. -### I20. Canonical execution offsets are an atomic projection of valid history - -- **Holds:** `sequenced_l2_txs` remains the append-only physical replay/audit - log. `executed_inputs` is a separate sparse projection for the current valid - history: every user op and non-batch-submitter direct input that executes has - exactly one mapping from its physical row to the pre-execution - `ExecutedInputCount`; batch envelopes and cockroach-root cursor-padding rows - have none. Current mappings occupy the contiguous logical interval `[K, H)`. -- **Creation atomicity:** a user-op chunk inserts its `user_ops`, trigger-created - physical rows, and explicit execution mappings in the same FULL transaction - that authorizes acknowledgements. A slow reconciliation turn inserts its - direct physical rows, mappings, frame rotation, and any snapshot promotion - in one transaction. The lane carries offsets attached to executed values, so - an included input cannot be persisted without its receipt. -- **Recovery semantics:** suffix invalidation retains physical audit rows but - deletes their derived mappings in the same transaction that advances - `RecoveryGeneration` and opens the replacement Tip. This rewinds `H` - naturally; replacement inputs reuse the suffix offsets under the new - generation. The global logical UNIQUE constraint and next-offset trigger - make a duplicate, gap, or out-of-order creation fail loud. Cockroach padding - stays outside the projection, and the durable safe-input floor prevents it - from being attributed later. -- **Snapshot/replay agreement:** every pending/finalized snapshot row stores - both physical `l2_tx_index` and canonical `executed_input_count`. Snapshot - registration asserts its count equals storage-derived `H`; startup compares - the loaded application's count with the row; catch-up then checks each - physical row's expected mapping before executing it, and replays each - user op with the persisted `frames.fee`, so the fee charged at replay is - the one inclusion-time execution used (catch-up re-executes through - `execute_valid_user_op` and does not re-validate). - Missing, extra, or wrong mappings are terminal invariant failures, never - repaired/backfilled. -- **Enforced by:** `ExecutedInputCount` receipts - (`sequencer-core/src/application/mod.rs`); attributed lane/storage APIs - (`ingress/inclusion_lane/`, `storage/ingress.rs`, - `storage/mutations.rs`); `executed_inputs` constraints and invalidation - trigger (`storage/migrations/0001_schema.sql`); storage-derived `H` - (`storage/history.rs`); snapshot count checks - (`storage/snapshot_dumps.rs`); coherent egress bounds/pages with contiguous - count checks (`storage/egress/canonical.rs`); and pre-execution catch-up checks - (`ingress/inclusion_lane/catch_up.rs`). -- **Depended on by:** restart determinism, standard-recovery rollback/reuse, - post-cockroach continuation at `K`, snapshot coherence, and the future - canonical-offset HTTP/WS protocol. -- **Breaks:** the same numeric offset can name the wrong application input, or - a restart can apply a different prefix than live execution—silent mirror or - canonical-state divergence. -- **Performance boundary:** deriving `H` is a covering lookup over the logical - UNIQUE index. Recovery deletes only its doomed projection suffix; it does - not scan invalid physical history on every hot-path insertion. Direct - execution receipt accumulation and classification live in the already-slow - L1 reconciliation regime; the user-op hot path adds one chunk-level mapping - query and inserts inside its existing durability transaction, not another - fsync or actor. +### I20. Application history is committed with its execution receipts + +- **Holds:** `application_inputs` contains every current included user op and + external direct exactly once, keyed by mandatory pre-execution count. Its + source reference, owning batch/frame, and payload tables reconstruct replay. + There are no entries for batch envelopes or the opaque baseline prefix. +- **Creation atomicity:** user-op source rows and application rows commit in the + same FULL chunk transaction that authorizes acknowledgements. Direct inputs + commit with the complete frame rotation, after checking all execution receipts. + Startup/recovery create leading direct rows before restoring the engine; + successful catch-up is required before admitting that sequence. +- **Recovery:** source records remain, while invalidation deletes the current + suffix. Replacement rows reuse counts under the incremented generation. +- **Snapshot/replay agreement:** a batch-close snapshot records storage-derived + `H`; the restored engine must report the same count. Each replay row must + match the engine's next count and executes with its persisted frame fee/clock + (or the direct input's source block). Missing rows or count mismatches fail + loud; they are never repaired or backfilled. +- **Enforced by:** shared execution receipts, storage append APIs, PK/FK/XOR/ + uniqueness and contiguous-offset triggers, coherent canonical pages, and + catch-up checks. +- **Performance boundary:** head discovery uses the integer primary-key maximum; + replay seeks directly by offset and joins bounded source rows. Neither scans + invalidated history. Chunk insertion adds no durability transaction or actor. diff --git a/docs/plans/2026-07-coordination-tracks.md b/docs/plans/2026-07-coordination-tracks.md index 49459d97..7509931c 100644 --- a/docs/plans/2026-07-coordination-tracks.md +++ b/docs/plans/2026-07-coordination-tracks.md @@ -14,7 +14,7 @@ freely at this stage — no backward-compatibility constraints. |---|-------|-------|--------| | 1 | WS context fields + L1 provenance (PR #26) | Stephen | **done** — merged to main | | 2 | Restore `docs/review/` ledger + this plan | us | **done** | -| 3 | Feed & replay protocol redesign | us (design) → us/Stephen (impl) | **internal read foundation implemented; consumer API open** — the [Track 3 ordered handoff](2026-07-track3-feed-replay-design.md#7-ordered-implementation-handoff) owns the accepted workflow and remaining cutover | +| 3 | Feed & replay protocol redesign | us (design) → us/Stephen (impl) | **implemented** — canonical application history, snapshot restore archives, mandatory WS claims, typed refusals, and SDK cutover; [remaining integration gates](2026-07-track3-feed-replay-design.md#5-acceptance-evidence-and-remaining-work) | | 4 | Storage decode policy | us | **done** — fail-loud for contract-impossible values; the named `saturating_query_bound` only where clamping preserves the predicate (policy lives in `storage/convert.rs` + the invariants check policy) | | 5 | Fee exponentiation LUT | us | **deferred** — decided exact-floor if built (the table *is* the spec, algorithm-free; replay continuity across the upgrade explicitly not preserved); a separate pending design decision may make log-space fees defunct — revisit after syncing with Bart | | 6 | Dump / `Application` API redesign | us + Bart | **revised interface implemented** — [Application contract](../protocol/application-contract.md); native bridge conformance is a separate integration branch | @@ -23,13 +23,13 @@ freely at this stage — no backward-compatibility constraints. **Current campaign order:** -1. Review the Track 3 internal history-read and snapshot-metadata foundation. -2. Implement the coordinated Track 3 HTTP/WS/SDK cutover on its successor branch. -3. Validate Track 6 against the reference C bridge, then the private DEX engine when shared. +1. Review and validate the integrated application-history, snapshot, and Track 3 cutover. +2. Validate Track 6 against the reference C bridge, then the private DEX engine when shared. +3. Run recovery/watchdog end-to-end gates and remeasure feed latency in the representative environment. 4. Track 5 (fee LUT) only after the log-space-fees decision. -Deferred (revisit with libdex rollout): multi-file/tar snapshot serving -(`docs/snapshots/lifecycle.md` known limitation), pending-snapshot-pool cap. +Full restore archives now support file and directory application prefixes. +Additional snapshot retention or transport mechanisms require a measured consumer need. ## Track 3 — Feed & replay protocol redesign @@ -40,16 +40,17 @@ claims, typed refusals, resource bounds, and fresh-snapshot recovery workflow. Raw `/inputs` and separate HTTP transaction replay are outside this feature. The watchdog retains its independent trusted-state/L1 comparison workflow. -The internal foundation provides typed history claims and policy errors, -coherent history-bound reads, inclusive canonical pagination, and history -identity captured with a snapshot's lease and count. Existing HTTP/WS responses -still expose physical cursors. The consumer cutover updates snapshot metadata, -WS admission/replay, SDK, and harness together; it must also remove the total -catch-up cap while retaining bounded pages, queues, and subscriber counts. - -Close the consumer invalidation finding only after that cutover and its -recovery/bootstrap acceptance tests. The next PR must demonstrate cold start, -ordinary resume, fresh-snapshot recovery, and gap-free backlog-to-tip delivery. +The implemented path uses one current `application_inputs` projection for catch-up +and egress. Snapshot headers identify the same leased artifact being downloaded; +WS claims name an era, generation, and inclusive next-input count. A valid +available backlog is replayable without a total catch-up cap, with bounded pages, +queues, and subscribers. Recovery refuses old claims before delivering inputs. + +The former physical replay cursor and sparse attribution design are superseded +by the [application-history design](application-history.md). Native-engine +bootstrap, representative latency measurements, and environment-dependent +recovery/watchdog runs remain integration gates; no additional protocol layer +is assumed for them. ## Track 5 — Fee exponentiation LUT (deferred) diff --git a/docs/plans/2026-07-track3-feed-replay-design.md b/docs/plans/2026-07-track3-feed-replay-design.md index 95c5a328..6da85a27 100644 --- a/docs/plans/2026-07-track3-feed-replay-design.md +++ b/docs/plans/2026-07-track3-feed-replay-design.md @@ -1,344 +1,129 @@ -# Feed & Replay Protocol Design (Track 3) - -**Status: internal read foundation implemented; consumer API cutover pending.** -Typed history claims and refusals, coherent canonical pages, and history -identity captured with snapshot leases are implemented alongside the durable -history/execution foundation. HTTP snapshot metadata, the canonical-coordinate -WS protocol, and the matching SDK/consumer workflow remain to be implemented. The current API contract is -in the [README](../../README.md); the [ordered handoff](#7-ordered-implementation-handoff) -defines the remaining work. Close the WS invalidation-contract finding only -after the consumer cutover and its acceptance tests. - -## 1. Consumer workflow and scope - -A subscriber starts cold by downloading an application-defined snapshot over -HTTP, restores its replicated application, and subscribes from that state's -executed-input count. One WS stream supplies every subsequent committed input -in order, then continues following the tip. Recovery and rebuild boundaries -must be detected before the consumer applies inputs from a different history. -The application owns the restore artifact and its format; egress transports -that artifact with enough metadata to identify the state it contains. - -The watchdog has a different trust boundary: it starts from trusted state, -advances independently using L1 inputs, and fetches the matching finalized -comparison artifact. Its finalized-state HTTP endpoints remain available. -The egress API serves both consumers exclusively within the operator's own -infrastructure, with network access controls. - -HTTP fits a finite snapshot download and already has file streaming and leases. -WS fits the existing ordered-input feed: stored and newly committed inputs use -the same replay loop. Splitting finalized replay onto HTTP would add another -input-fetching path and a moving-boundary handoff without helping this replica -workflow. Raw `/inputs` and paginated HTTP `/l2-txs` are outside this scope; -revisit them for a concrete archival or historical-query consumer. WS carries -backlog on both sides of the gold boundary. - -The current snapshot-then-subscribe path already supports ordinary bootstrap, -but its physical rowid cursor cannot detect recovery discontinuities. Its total -catch-up limit can also reject a valid snapshot whose download/restore or -preceding execution leaves too much backlog. The new protocol addresses those -boundaries without requiring different checkpoint timing. - -## 2. Concepts and coordinates - -- **Input-box coordinate `input_index: u64`** — position in `safe_inputs` - (per-application InputBox order). Append-only and sourced from L1 safe blocks. -- **Feed coordinate `offset: ExecutedInputCount`** — the authoritative - `Application::executed_input_count()` boundary, starting at zero. An - application at `X` is ready to consume history entry `X`; applying that - entry advances it to `X + 1`. SQLite stores this as a sparse canonical - attribution beside its append-only physical rowid replay log. The current - feed still exposes rowid and changes only at the API cutover. -- **Era base `K`** — the smallest feed offset locally available in this era. - Genesis setup starts at zero. Cockroach recovery sets it to - `S'.executed_input_count()` after the fold; absolute offsets continue, but - the unavailable prefix is not reconstructed. `K` is application history, - not the snapshot's physical `l2_tx_index`: recovery cursor-padding rows may - advance the latter without executing an application input. -- **Snapshot count `C`** — the exact executed-input count in a selected - application dump. The replica restores at `C` and requests input `C` next. - A current snapshot may be newer than `K`; retaining a snapshot exactly at - `K` is not required. -- **Gold boundary `G`** — within one era, the exclusive executed-input count - after the scheduler-accepted prefix. Entries with `offset < G` cannot be - invalidated; `G` only advances. It does not restrict WS admission or - determine which transport carries an input. -- **Era ID `e`** — random durable UUIDv4 minted write-once in one era's - baseline transaction. Cockroach recovery/fresh setup creates a new era - because the rebuilt DB cannot serve the prior era's ordered L2 history from - genesis. -- **Recovery generation `g: u64`** — soft-suffix reality version within one - era. Bumped exactly once by a standard-recovery transaction iff it - invalidates at least one valid batch. Entries with `offset < G` are - generation-free within that era. -- **History version `(e, g)`** — equality/discontinuity token carried by the - protocol. It is not a globally ordered number. -- **Live head `H`** — the current application's exclusive - `executed_input_count`; locally available entries occupy `[K, H)`. - -## 3. History-version semantics - -- `EraId` is a 16-byte UUIDv4 newtype, persisted write-once per era and exposed - as canonical lowercase hyphenated JSON. Store `created_at` separately; a - bare timestamp is not collision-resistant under clock rollback or - simultaneous setup. -- `RecoveryGeneration` starts at zero and increments exactly once in the same - transaction iff standard recovery invalidates at least one valid batch. - Ensuring/reopening a missing Tip without invalidation does not bump it. -- Clean restart and inspection that admits without changing history change - neither field. -- Cockroach recovery/fresh setup mints a new era and resets generation to - zero. An interrupted attempt that retains its incomplete DB reuses the - already-minted, externally unexposed era. A fail-loud partial-fill refusal - requires an operator wipe/retry and therefore mints another unexposed era. - A new era is an explicit operator-driven setup/rebuild action; there is no - in-place rotation tool, automated DB replacement, implicit clone detection, - distributed fencing, or partial-fill resume protocol. -- Copying an initialized DB copies its era too. Arbitrary clone-and-run is - unsupported. Operating copied state as a new era requires explicit - fresh/wiped-directory setup/rebuild; detecting uncoordinated clones requires - external authority. -- The snapshot response carries the pair together with the dump's count; - every subscribe request claims that pair. WS responses carry the admitted - recovery generation. The bootstrap workflow requires no separate - `/history-version` request. Reading current metadata cannot authorize an old - state: the client keeps the history identity associated with its own state. - -Standard recovery restores the retained application state, so its count rolls -back, then advances over replacement force-drained directs. The same suffix -count may name a different input under a new generation. The initial client -workflow handles both a stale generation and a changed era by discarding its -replica and downloading a current snapshot. It does this even when the old -state's numeric count is in the new history's range. Retaining a known-stable -checkpoint for cheaper rollback is a possible later client optimization. - -Recovery and rebuild run before runtime admission across a process boundary. -Existing subscriptions end before history changes. Each admitted session is -bound to its validated history version; reconnect validation is the correctness -boundary. A guaranteed farewell, invalidation broadcaster, or generation-polling -worker is unnecessary under this lifecycle. - -Cockroach recovery leaves the rebuild base NULL at baseline creation, then -binds `K = S'.executed_input_count()` in the same transaction that registers -the initial finalized snapshot. Setup completion refuses until both exist. -Requests below `K` receive a typed `history_unavailable` response carrying -`available_from = K` and the bootstrap recipe. This preserves the absolute -application coordinate without claiming that the rebuilt DB can serve the -lost prefix. - -### 3.1 Landed storage representation - -The physical and logical coordinates deliberately remain separate: - -- `sequenced_l2_txs.offset` is the append-only SQLite replay/audit cursor. - Invalidated rows remain, and batch-envelope/cockroach-padding rows exist even - though the application does not execute them. -- `executed_inputs` is a sparse **current-canonical projection** from an - executable physical row to its pre-execution `ExecutedInputCount`. User-op - and direct mappings commit atomically with their existing durability - transaction; envelopes and padding have no row. -- Standard recovery retains physical audit history but deletes the invalidated - mapping suffix in the same transaction that bumps generation and opens the - replacement Tip. `H` therefore rolls back without scanning invalid physical - history, and replacements reuse the same suffix offsets under the new - generation. -- Snapshot rows store both physical `l2_tx_index` and canonical - `executed_input_count`. Registration checks the app count against - storage-derived `H`; startup checks the loaded dump against the snapshot row; - catch-up checks every mapping before executing its physical row. - -There is no backfill, repair, or neighbor-derived fallback. A missing, extra, -or wrong attribution is a terminal self-invariant failure. This keeps the -API cutover a projection over already-correct durable values rather than -the moment those values first become authoritative. - -## 4. HTTP snapshot bootstrap - -`GET /latest_snapshot` supplies the latest available application snapshot: -latest valid pending snapshot if present, otherwise finalized. The response -carries the selected dump's canonical count `C` and history version `(e, g)`. -These are selected together with the artifact's lease in one coherent storage -transaction. They describe that artifact at acquisition, not a separately read -live head after the download. - -Logical response metadata, with header spellings fixed during the API cutover: - -```json -{ - "era_id": "550e8400-e29b-41d4-a716-446655440000", - "recovery_generation": 7, - "executed_input_count": 100 -} -``` - -The response body is the application-defined restore artifact. The current -handler opens the single file named by `Application::state_file_in_dump`. -Integration must demonstrate that the delivered artifact restores the intended -replica, including progress; the application contract permits recovery dumps -whose full representation differs from that comparison file. This is an -application/adapter integration obligation, not an egress-owned state format. - -The lease protects the artifact through response completion or disconnect. -Once downloaded, the consumer owns its copy and does not depend on the server -retaining that dump. A snapshot exactly at `K` need not remain available: a -retained snapshot at `C >= K` initializes the replica, which replays from `C`. -Recovery during download or restoration can invalidate the claim; subscription -validation handles that race without holding intake or history advancement. - -The client verifies the restored application's count against `C`. Cache -validators must distinguish the era and selected artifact, including a rebuilt -era at the same inclusion block. Cached bytes must retain their matching -history metadata; a fresh version lookup must not relabel cached old state. -Any conditional response must preserve that association. Range resumption is a -possible later transport feature, not a prerequisite for cold bootstrap. - -`GET /finalized_state` and its metadata route keep serving the watchdog's -comparison workflow. Their artifact metadata and cache identity must likewise -refer to the selected finalized checkpoint. The watchdog's trusted starting -state and independent L1 replay are not replaced by the tip-replica bootstrap. - -## 5. WS subscription v2 - -### 5.1 Admission and continuity - -`GET /ws/subscribe?from_offset=N&era_id=e&recovery_generation=g` requires all -three coordinates. The client claims the history associated with its own state -and the exact application boundary it is ready to execute. Validate the claim -against one coherent read of `(e, g, K, H)` before delivering any input. - -Apply history checks before offset checks: - -| Condition | Response | Client action | +# Feed and Replay Protocol (Track 3) + +**Status: implemented in the current application-history redesign.** +The former physical-rowid feed and sparse execution mapping are superseded. +The [README](../../README.md) owns the wire contract; the +[application-history design](application-history.md) owns storage and recovery +boundaries. This document records the consumer workflow and remaining gates. + +## 1. Consumer workflow + +1. Download `GET /latest_snapshot` using the SDK. Its tar body contains the + complete immutable restore artifact (`info.toml` and the opaque `state` + file or directory). +2. Restore the application and verify its executed-input count against + `X-Executed-Input-Count`. Keep the matching `X-History-Era` and + `X-Recovery-Generation` headers with those bytes and the restored state. +3. Subscribe with that `HistoryClaim`: mandatory `era_id`, + `recovery_generation`, and `next_input` query fields. +4. Apply each entry whose `offset` equals the application's current count. + Successful execution advances the count by one. Persist identity with the + replicated state before using it for a later resume. +5. After an ordinary disconnect, reconnect with the saved identity and actual + next-input count. On an era or generation refusal, discard the incompatible + replica and bootstrap from a current snapshot. + +A fresh identity lookup cannot authorize old state. The SDK requires an explicit +claim on every subscription; it does not silently change identity on reconnect. +There is no separate history-version endpoint. + +Snapshot selection, its count, history version, and GC lease share one storage +transaction. The lease lasts through response completion or disconnect. A +recovery between snapshot acquisition and subscription is handled by refusing +its old claim, without blocking history advancement during transfer. + +## 2. Coordinates and storage + +- `safe_inputs.safe_input_index` names a source L1 InputBox event. It includes + scheduler batch envelopes and direct application inputs. +- `application_inputs.offset` is an `ExecutedInputCount`: an application at + count `N` consumes entry `N` next. Every row has an offset, owner frame, and + exactly one user-op or source-L1 reference. Batch envelopes never appear. +- The immutable era baseline supplies the unavailable application prefix + `K` and accounted L1 block. Current entries occupy `[K, H)`, where `H` is + the next application count. +- Standard recovery deletes an invalidated projection suffix and advances + its generation atomically. Replacement inputs reuse those canonical + offsets. Raw L1 inputs, batches, frames, and user ops retain their source + evidence; the invalidated flattened sequence is not separately retained. +- Rebuild creates a new UUIDv4 era and a complete baseline. It does not insert + padding inputs or preserve a physical replay cursor. + +Catch-up and egress use the same named entry and coherent canonical-page +reader. Bounds, identity, and rows are read in one SQLite transaction. Missing +interior rows and invalid payload context fail loudly. Empty requests at the +head do not convert the exclusive boundary back into a SQL row coordinate. + +The latest valid frame's `safe_block`, bounded below by the era's L1 baseline, +accounts for the complete L1 prefix. No separate mutable processed-input cursor +is needed. Snapshot application count and L1 accounting are different facts; +see the application-history design for recovery's terminal drain and sparse +checkpoint availability. + +## 3. Subscription admission + +Validate history identity before position, using one coherent `(era, +generation, K, H)` read: + +| Condition | HTTP 409 policy code | Consumer action | |---|---|---| -| Era differs | `era_changed` | Download and restore a current snapshot. | -| Generation differs | `stale_generation` | Download and restore a current snapshot. | -| `N < K` | `history_unavailable`, with `available_from = K` | Download and restore a current snapshot. | -| `N > H` | `ahead_of_head`, with `live_head = H` | Report the invalid claim; do not wait or silently clamp it. | -| `K <= N <= H` | Admit | Replay inclusively from `N`, then follow new inputs. | - -Policy refusals use a typed error response before any data, followed by close -1008. Successful admission sends `hello` with the admitted recovery generation -and coherent available/head boundaries. Exact response fields and header names -are fixed together with the SDK during the cutover. The era is bound by the -mandatory request claim; every message on an admitted session carries that -session's generation. Refusal metadata describes the history observed when -validating the rejected claim. - -### 5.2 Replay and resource bounds - -Every matching-history offset in `[K, H]` is serveable. For `N < H`, the first -returned input is entry `N`. A request exactly at `H` waits for the next input. -The gold boundary does not gate admission: finalized and soft entries use the -same ordered stream, and advancing finalization requires no client action. - -Read canonical pages from committed valid SQLite history, with a bounded page -size and bounded send queue. Preserve the concurrent-subscriber limit. There -is no total catch-up event cap: neither snapshot cadence nor download/restore -time guarantees a backlog below 50,000 inputs, and refetching can return the -same snapshot repeatedly. Memory and concurrency remain bounded independently -of total history depth. A replica must process faster than ongoing production -to reach the tip; another transport cannot remove that capacity requirement. - -The client verifies that each event's offset equals its application's current -count, applies the input, and advances that count. It resumes from the next -unapplied input, not the last received network message. A client that persists -its replica must preserve the corresponding history identity with that state. -An offset mismatch is a continuity failure; skipping an input or jumping to a -suggested live head would corrupt the replica. - -### 5.3 Snapshot-to-tip walkthrough - -1. Download an artifact with `(era=A, generation=7, count=100)` and restore it. -2. Subscribe with `(A, 7, from_offset=100)`. If the head is 105, receive entries - `100..104`, including any entries already finalized. -3. Continue on the same socket when input 105 commits; no transport handoff or - special catch-up transition is required. -4. After an ordinary disconnect, reconnect using the state's saved history - version and actual next-input count. Clean restart preserves that version. -5. If recovery changes the generation, or a rebuild changes the era, the old - claim is rejected before data. Discard the replica, download a current - snapshot, and repeat. This also handles recovery between steps 1 and 2. - -### 5.4 Events and wire format - -Keep the existing denormalized user-op/direct-input context: nonce, fee, safe -block, batch nonce, input index, block timestamp, and transaction hash where -applicable. Replace physical rowids with canonical next-input coordinates. -JSON text messages share a tagged SDK/server enum containing input events, -`hello`, and typed `error` responses. A live-transition message, frame/batch -boundary events, or best-effort invalidation message requires a concrete -consumer need; the basic replay loop and reconnect rules do not rely on them. - -## 6. Compatibility boundary - -The cutover changes snapshot metadata and the subscription contract together -with the SDK and replica harness. Mandatory history claims and canonical -inclusive offsets replace the optional physical-cursor subscription. Remove -the total catch-up rejection and its suggestion to skip directly to the live -head. An ordinary disconnect permits a same-version resume; recovery requires -a new snapshot in the initial client workflow. - -Keep the deployed README truthful until implementation lands. No intermediate -generation-aware physical-rowid API is needed. The storage foundation alone -does not close the consumer invalidation finding. - -## 7. Ordered implementation handoff - -The [coordination roadmap](2026-07-coordination-tracks.md) -owns PR sequencing. Implement two review boundaries: - -1. **Internal read and snapshot foundation (implemented).** Define typed history claims, - canonical pages, snapshot metadata, and policy errors. Read `(e, g, K, H)` - coherently, paginate inclusively through `executed_inputs`, and acquire the - snapshot's count/version with its artifact lease. Preserve physical cursors - for internal catch-up and recovery. Keep intermediate helpers internal. - The unused internal canonical reader has a scoped non-test dead-code - expectation until the next step gives it a runtime caller; remove that - expectation when connecting WS. -2. **Coordinated consumer cutover (next).** Project snapshot metadata over HTTP and - require `(EraId, RecoveryGeneration, ExecutedInputCount)` on WS. Admit the - entire available history range, remove the total catch-up cap, and implement - typed refusals and fresh-snapshot remediation. Update SDK, replica harness, - cache validators, and consumer documentation in the same deployable change. - Fix the exact metadata/error serialization as part of that shared contract. - -Required evidence belongs with the change that owns the behavior: - -- Coherent metadata and canonical pagination across physical-row holes, - envelope/padding rows, nonzero era bases, and replacement suffixes. -- Snapshot bytes, count, and version remain associated through transfer and - cache reuse; restored application count agrees with response metadata. -- Clean restart and ordinary reconnect resume without rebootstrap. -- Stale generation is refused before any data; replacement inputs can reuse - canonical counts without preserving the invalidated replica. -- A changed era is detected even with the same numeric generation/count, and - cache validators differ for rebuilt artifacts at the same inclusion block. -- Below-base and ahead-of-head claims receive their typed errors; a claim at - the head waits normally, and a valid claim below gold is admitted. -- More than 50,000 inputs after the latest snapshot, including direct-heavy - history, remain replayable with bounded pages and queues. -- Writes during download/restoration and replay-to-live delivery cause no gaps; - recovery between snapshot acquisition and subscription forces rebootstrap. -- Subscriber limits, disconnect cleanup, snapshot lease lifetime, and watchdog - finalized comparison behavior remain correct. - -Remeasure submit-to-matching-WS-event latency, rewrite the README, graduate the -normative protocol text into `docs/protocol/`, and close the register's WS -invalidation-contract finding only after the cutover and acceptance evidence. -Feed output comes from committed valid SQLite history matching the admitted -history version; it does not depend on a global runtime actor. - -## 8. Revisit triggers - -- **Archive or historical queries:** evaluate dedicated HTTP replay endpoints - when a consumer needs capabilities beyond snapshot-to-tip replication. -- **Recovery cost:** consider retaining a known-stable client checkpoint when - repeated full snapshot restoration is a measured problem. -- **Snapshot transport:** add resumable transfer or another artifact packaging - only when actual application size/layout and clients require it. -- **Extra stream controls:** add live/frame/batch boundary events only for an - identified consumer operation that cannot use the existing event context. -- **Access boundary:** authentication and public rate-limit policy require a - separate decision if egress is exposed beyond operator infrastructure. -- **Runtime lifecycle:** re-evaluate session fencing if recovery can change - history inside an admitted process or multiple writers are introduced. +| Era differs | `ERA_CHANGED` | Bootstrap from a current snapshot. | +| Generation differs | `STALE_GENERATION` | Bootstrap from a current snapshot. | +| `N < K` | `HISTORY_UNAVAILABLE`, with `available_from` | Bootstrap from an available snapshot. | +| `N > H` | `AHEAD_OF_HEAD`, with `head` | Correct the invalid claim. | +| `K <= N <= H` | Upgrade to WebSocket | Replay inclusively from `N`, then follow the tip. | + +Refusals precede the upgrade and all input delivery. The JSON body is also +carried in `X-History-Error`: WebSocket libraries may stop reading a refused +handshake at its headers before the body arrives. Missing or malformed required +query fields receive HTTP 400. + +A successful stream carries the existing tagged user-op/direct-input messages +with canonical offsets and persisted context. The mandatory admission claim +binds the session identity; there is no hello frame or per-event generation. +Recovery changes history only across a process boundary, after existing +subscriptions have ended. No generation bus or farewell guarantee is needed. + +`N == H` waits normally. Every valid available backlog is replayable: there is +no total 50,000-event cap. Page size, send queue, subscriber count, and inbound +message limits remain bounded independently of backlog depth. The same durable +query handles backlog and live delivery, avoiding a separate handoff cursor. + +## 4. Snapshot and watchdog boundaries + +`/latest_snapshot` is a replica restore archive. `/finalized_state` remains the +watchdog's application comparison bytes, with its inclusion-block metadata +route. `/finalized_snapshot` exports an accepted recovery artifact and a derived +`checkpoint.toml` receipt. These are operator-infrastructure routes. + +The watchdog starts from trusted state and independently consumes L1; snapshot +bootstrap for a tip replica does not replace that trust boundary. Finalized +comparison/export is available only at a supported accepted checkpoint, not at +an invented intra-frame or arbitrary execution position. + +## 5. Acceptance evidence and remaining work + +The implementation tests cover inclusive pages and source context, exclusion +of envelopes, nonzero rebuild bases, actual suffix invalidation and replacement, +coherent SQLite snapshots during a second writer's recovery, counts beyond the +largest SQL row, and loud interior-gap detection. Feed/API/SDK tests cover +mandatory claims, typed refusals, exact-head waiting and live delivery, +50,001-entry history with bounded pages, ordinary resume, subscriber limits, +terminal storage faults, and cancelled preparation retaining process ownership. +Snapshot integration tests own artifact/header association and restore proof. +The Anvil recovery/WS gate also exercises process restart, generation refusal, +and re-drained direct replay at a reused offset. + +Remaining integration gates are concrete consumers and environments: + +- Validate the native reference adapter and, when available, the private DEX + bridge against the application contract and this bootstrap workflow. +- Remeasure submit-to-matching-WS-event latency on the representative deployment. +- Complete the broader recovery/watchdog scenarios in the pinned emulator + environment. The native Anvil recovery gate does not prove canonical-machine + comparison; the local host currently has emulator 0.21 while the repo pins 0.20. + +Revisit resumable snapshot transfer only when artifact size requires it; +retained client checkpoints only when full rebootstrap cost matters; archival +HTTP replay only for an identified consumer. Revisit session fencing if history +can mutate within an admitted process or multiple writers become supported. diff --git a/docs/plans/2026-08-authority-boundary-adr.md b/docs/plans/2026-08-authority-boundary-adr.md index 883eeda3..72b78965 100644 --- a/docs/plans/2026-08-authority-boundary-adr.md +++ b/docs/plans/2026-08-authority-boundary-adr.md @@ -132,7 +132,7 @@ returning to the outer loop costs only time-gate bookkeeping — no fsync and no frontier read per chunk. The **L1 reconciliation regime** fires when the observed safe head is at least five blocks past the open frame's clock: it consumes the complete accumulated newly-safe range, catch-up and backlog -conditions included, promotes at most once, and opens exactly one frame at +conditions included, and opens exactly one frame at the observed tip — jumps are never interpolated. There is no elapsed-time budget, preemption, or resumable partial cursor inside a turn: the supported deployment assumes the application promptly digests the whole range @@ -142,7 +142,7 @@ only if production measurements disprove that). Authority remains role-local and auditable: a FULL-committed user-op chunk authorizes its acknowledgement; a valid sealed batch plus the durable write-before-broadcast watermark authorizes an L1 submission; committed -valid physical replay rows authorize the current feed output. Effects handed to the network before process termination may still +version-checked application-input rows authorize the feed output. Effects handed to the network before process termination may still complete remotely. ## Rejected alternatives @@ -168,11 +168,10 @@ HistoryPosition = (HistoryVersion, ExecutedInputCount) setup/rebuild era; `RecoveryGeneration` increments exactly once in the standard-recovery transaction iff it invalidates at least one valid batch; a clean restart changes neither. The pair is an equality/discontinuity token, -not an ordered counter. The durable canonical-coordinate foundation is -landed ([I18](../invariants.md), [I20](../invariants.md)). The current public -feed still uses physical SQLite rowid offsets; replacing them with -`ExecutedInputCount` and exposing history versions is owned by the -[Track 3 handoff](2026-07-track3-feed-replay-design.md#7-ordered-implementation-handoff). +not an ordered counter. Snapshot headers and mandatory WS claims expose these coordinates. Every +application row has its pre-execution count; recovery replaces only the current +suffix. See the [history contract](application-history.md) and +[Track 3 handoff](2026-07-track3-feed-replay-design.md). ## Performance posture diff --git a/docs/plans/application-history.md b/docs/plans/application-history.md new file mode 100644 index 00000000..e7b935f6 --- /dev/null +++ b/docs/plans/application-history.md @@ -0,0 +1,92 @@ +# Application history and checkpoints + +The sequencer keeps L1 observations, application ordering, and batch acceptance +as separate durable facts. L1 inputs include batch envelopes; application +history contains only included user operations and external direct inputs. +Source references provide provenance without defining a mapping between the +two timelines. + +## History + +`application_inputs` is the current application sequence. Its primary key is +the input's pre-execution `ExecutedInputCount`; each row belongs to a local +batch/frame and references either its user operation or its source L1 input. +Every row executes. Payloads remain in their source tables. + +The latest surviving frame's `safe_block` records complete L1 accounting. +Reconciliation covers the whole newly safe interval before committing its new +frame and application inputs. Full-block ingestion and indivisible range +reconciliation make a separate mutable processing cursor unnecessary. Empty +intervals and intervals containing only batch envelopes advance this boundary +without adding application inputs. + +Recovery invalidates a batch suffix, removes its current application rows, +advances the history generation, and opens the replacement tip atomically. +Replacement inputs reuse suffix offsets under the new generation. Original +L1, batch, frame, and user-operation records remain available for diagnostics. + +## Era baseline + +Setup registers a complete baseline after its artifact is durable: application +count `K`, accounted L1 stop block `C`, starting batch nonce, and history identity. +The recovered L1 prefix through `C` is opaque to ordinary operation. Both direct +ordering and accepted-batch scanning begin after it. The baseline metadata +survives root invalidation and artifact garbage collection. + +The recovery fold drains queued directs through `C`, including young directs +that the canonical scheduler has not executed yet. Its output is a restart +baseline, without a claim that its bytes equal canonical state at block `C`. +Genesis supplies the trusted block-zero comparison state. + +## Snapshots and acceptance + +The lane creates a durable snapshot at every batch close. Snapshot registration +and batch sealing commit together. Snapshots reference immutable local batch +identities; a nonce can be reused by recovery. The baseline is a separate +snapshot origin. + +Acceptance is derived from complete safe L1 observations, the scheduler's +acceptance rules, and byte identity with the local sealed batch. An accepted +batch confirms existing application history and adds no replay entry. + +Checkpoint selection uses these facts directly: + +- Restart and replica bootstrap use the newest surviving batch snapshot, or + the baseline. +- Recovery requires a retained accepted snapshot, or the baseline before the + first post-baseline acceptance. +- The watchdog compares an accepted checkpoint at the end of its L1 inclusion + block. Per-batch snapshots make the latest accepted batch's artifact available. + +Select the required accepted batch before loading its snapshot: a missing +required artifact is an invariant violation, never permission to choose an +older checkpoint. Divergence blocks publication of a newly derived comparison. + +There is no snapshot promotion mutation. Retention keeps the newest accepted +snapshot (or baseline), all valid snapshots beyond the accepted frontier, and +leased artifacts. The baseline bytes can be retired once an accepted artifact +provides the recovery fallback. Artifact creation precedes DB publication; +DB retirement precedes filesystem deletion. + +A portable accepted checkpoint includes the application artifact and coherent +sequencer metadata identifying its canonical comparison point and resume nonce. +Acceptance metadata is derived at export; application artifacts stay immutable. +Sparse snapshot creation and intra-block watchdog checkpoints are separate work. + +## Replay and egress + +Restart and egress share application-only pages beginning at an inclusive input +count. Snapshot bootstrap uses HTTP; one WS stream replays available history +then follows the tip. Subscription claims include era, generation, and next +input count. Wrong identity or unavailable history requires bootstrap; a claim +at the head waits and one beyond it fails. Pages and queues are bounded, while +total replay has no arbitrary catch-up cap. + +## Validation boundaries + +Exercise prefix exclusion for previously rejected future-nonce batches; +baseline-only restart and repeated root invalidation; envelopes-only frame +advancement; acceptance observed during downtime; empty accepted batches; +snapshot retirement with active leases; atomic suffix replacement; and cold +replica restore followed by canonical replay. The recovery models constrain +admission and batch safety, not the concrete snapshot/GC implementation. diff --git a/docs/protocol/application-contract.md b/docs/protocol/application-contract.md index 8acd89df..40f0c8f1 100644 --- a/docs/protocol/application-contract.md +++ b/docs/protocol/application-contract.md @@ -115,9 +115,9 @@ cockroach recovery supplies an absolute starting count from the recovered engine. SQLite stores an independent expected snapshot count and per-input execution offsets, checked during catch-up. -The current HTTP/WS feed still uses physical SQLite rowids. History-version -and canonical-offset projection remain Track 3 work; clients must follow the -current README until that cutover. +HTTP snapshot metadata and mandatory WS claims carry the history version and +this count. Both restart and subscriber replay read the same current application +sequence; see the [API contract](../../README.md). ### 5. Operational capacity for L1 reconciliation @@ -126,7 +126,7 @@ accumulated input range the persisted frontier can expose in one L1 reconciliation turn, including backlog within the supported operating envelope. The lane processes that range before returning to user-op work. There is no elapsed-time cutoff, preemption, or durable timeout-and-resume -cursor. Paging may bound memory; the drain/promotion commit remains atomic. +cursor. Paging may bound memory; the complete-range reconciliation commit remains atomic. This is a deployment assumption. A request overlapping reconciliation or synchronous checkpoint creation may see extra acknowledgement latency. diff --git a/docs/protocol/c-application-binding.md b/docs/protocol/c-application-binding.md index ef3f6981..adc06e27 100644 --- a/docs/protocol/c-application-binding.md +++ b/docs/protocol/c-application-binding.md @@ -19,8 +19,8 @@ linking, genesis, host commands, and the reference wallet under `examples/`. defines record layout, statuses, pointer ownership, and call lifetimes. An engine archive and its generated Rust bindings must agree on that header. - [Snapshot lifecycle](../snapshots/lifecycle.md) owns checkpoint registration, - promotion, reader leases, and garbage collection. The application owns its - checkpoint representation within the supplied prefix. + acceptance-derived selection, reader leases, and garbage collection. The + application owns its checkpoint representation within the supplied prefix. ## Execution and ownership diff --git a/docs/protocol/scheduler-semantics.md b/docs/protocol/scheduler-semantics.md index bbf030c3..4d1550a0 100644 --- a/docs/protocol/scheduler-semantics.md +++ b/docs/protocol/scheduler-semantics.md @@ -179,11 +179,9 @@ suffix at the same application offsets; cockroach recovery resumes from the absolute count persisted in the recovered application state even when older history is no longer locally available. -> **Cutover status:** the typed execution boundary, scheduler count -> transitions, and durable per-input mapping are landed. Physical -> `sequenced_l2_txs.offset` remains SQLite rowid and the existing WebSocket -> still exposes that cursor; changing the public protocol to canonical offsets -> and `HistoryVersion` remains Track 3 work. +The durable `application_inputs` sequence uses these same offsets. HTTP +snapshots carry the history version and application count; WS subscriptions +must claim both before inclusive replay. See the [history contract](../plans/application-history.md). --- @@ -231,7 +229,7 @@ I1 names three places this algorithm lives. They are not three rewrites; two are site. **Why the agreement is load-bearing:** the gold frontier (#2) is what -recovery's cascade pivots on and what promotion trusts; the lane (#3) is what +recovery's cascade pivots on and what accepted-checkpoint selection trusts; the lane (#3) is what users see as soft confirmations. If any of the three computes a different accept/reject/order than the canonical fold (#1), the sequencer will have promised users a future the scheduler will not produce. No mechanism enforces diff --git a/docs/recovery/README.md b/docs/recovery/README.md index dd1d4f4e..32dc38fd 100644 --- a/docs/recovery/README.md +++ b/docs/recovery/README.md @@ -1,6 +1,6 @@ # Batch Recovery -This document describes the recovery design for the sequencer: how the system detects that batches are failing to land on L1, how startup recovers to a consistent state, and where runtime authority begins. Two complementary bounded TLA+ models cover the design: [`preemptive.tla`](preemptive.tla) for batch/slot safety and [`admission.tla`](admission.tla) for startup phase ordering and admission. They do not currently model the external era/generation/base metadata or the derived canonical `executed_inputs` projection; their crash atomicity is enforced by the SQLite transaction boundaries and schema triggers described below. +This document describes the recovery design for the sequencer: how the system detects that batches are failing to land on L1, how startup recovers to a consistent state, and where runtime authority begins. Two complementary bounded TLA+ models cover the design: [`preemptive.tla`](preemptive.tla) for batch/slot safety and [`admission.tla`](admission.tla) for startup phase ordering and admission. They do not currently model the external era/generation/base metadata or the canonical `application_inputs` projection or snapshot artifact/GC lifecycle; their crash atomicity is enforced by the SQLite transaction boundaries and schema triggers described below. See `AGENTS.md` "Batch Staleness and Recovery" for quick-reference tables and function names. @@ -260,9 +260,9 @@ After step 3 (flush) and step 4 (re-sync), the gold frontier is fresh. Run the a 1. **Find the cascade pivot.** First try the closed pivot: first valid closed batch with `nonce >= frontier_nonce`. By the contiguity invariant, this batch's nonce is exactly `frontier_nonce`. If one exists, cascade from it. 2. **No closed pivot? Check the Tip.** When all closed batches landed fresh and were accepted (the "everything worked" aftermath), there's no closed pivot — but the Tip can still be in the danger zone. When the lane rotates without a safe-block advance between frames (e.g. immediately after init, both frames share the bootstrap `safe_block`), `S_tip = S_closed`. The closed batch can become gold by inclusion-staleness while the Tip's age — measured against `current_safe_block` after the flush wait — has crossed the danger zone. Pure monotonicity (`S_tip ≥ S_closed`) doesn't rule this out: equality is allowed. So fall through to `find_tip_batch_in_danger(danger_threshold)`. If the Tip's age clears `danger_threshold`, cascade it. -3. **Cascade-invalidate the suffix**: set `invalidated_at_ms` on every valid batch with `batch_index >= pivot.batch_index`. This catches all non-gold batches in cases (2)/(3) above, and the Tip alone in the no-pivot-but-Tip-aging case. The invalidation trigger retains physical replay rows but deletes their derived `executed_inputs` mappings, rewinding canonical head `H` to the surviving prefix. -4. **Advance external history reality**: iff step 3 invalidated at least one valid batch, increment `RecoveryGeneration` exactly once in this same SQLite transaction. A no-invalidation repair does not bump it. Mapping rewind and generation change are therefore one visible transition. -5. **Open recovery batch**: parent is the last valid ancestor (`MAX(batch_index) FROM valid_batches` after the cascade). Nonce is structurally `parent.nonce + 1`, which equals `frontier_nonce` — the scheduler's `expected_nonce`. Re-drain direct inputs from the invalidated batches starting at `max(base_safe_input_index, MAX(valid safe_input_index) + 1)`. Their new physical rows reuse the rewound logical offsets under the incremented generation. +3. **Cascade-invalidate the suffix**: set `invalidated_at_ms` on every valid batch with `batch_index >= pivot.batch_index`. This catches all non-gold batches in cases (2)/(3) above, and the Tip alone in the no-pivot-but-Tip-aging case. The invalidation trigger deletes those batches' canonical `application_inputs` rows, rewinding head `H` to the surviving prefix. Raw L1, batch, frame, and user-op source facts remain available for audit. +4. **Advance external history reality**: iff step 3 invalidated at least one valid batch, increment `RecoveryGeneration` exactly once in this same SQLite transaction. A no-invalidation repair does not bump it. Application-history rewind and generation change are therefore one visible transition. +5. **Open recovery batch**: parent is the last valid ancestor (`MAX(batch_index) FROM valid_batches` after the cascade). Nonce is structurally `parent.nonce + 1`, which equals `frontier_nonce` — the scheduler's `expected_nonce`. Reconcile external directs after the latest surviving frame's `safe_block`, or immutable baseline block `C` if no frame survives. The new application rows reuse offsets beginning at the rewound head under the incremented generation. **Threshold = `danger_threshold`, not `MAX_WAIT_BLOCKS`**. We're already committed to recovery; the Tip is past gold; if it's also past the threshold that would have triggered recovery had it been a closed batch, cascade it. Otherwise the next danger detector tick after resume would re-trip on the Tip's eventual close + submission anyway (the closed batch would inherit its first frame's safe_block). @@ -276,7 +276,7 @@ Closed batches past gold (if any) are still in their natural lifecycle — pendi 2. Open a fresh recovery batch in the same transaction. 3. If no Tip in danger and no Tip exists at all (torn-state crash recovery), open a Tip anyway. -The `Safe` decision with no open Tip runs `EnsureOpenTip`. Its transaction rechecks `Safe`, finalized-snapshot presence, and Tip absence, then opens the Tip through `open_fresh_tip_in_tx`. It refuses rather than commit without an open Tip. Startup rechecks danger after this repair; Tip creation never occurs as a worker-construction side effect. +The `Safe` decision with no open Tip runs `EnsureOpenTip`. Its transaction rechecks `Safe`, rollback-safe checkpoint presence, and Tip absence, then opens the Tip through `open_fresh_tip_in_tx`. It refuses rather than commit without an open Tip. Startup rechecks danger after this repair; Tip creation never occurs as a worker-construction side effect. #### Why `danger_threshold`, not `MAX_WAIT_BLOCKS`, for the Tip threshold @@ -315,7 +315,7 @@ Each loop iteration burns gas (no-ops + doomed resubs), takes ~12 minutes (the f ### Startup behavior summary -Startup holds the exclusive process lock and launches no workers until recovery and preparation finish. Its first local inspection refuses canonical divergence or missing finalized state before any provider call. It then attempts one initial Sync: a provider failure may use a still-fresh persisted view, while other failures retain their typed retry/refuse classification. +Startup holds the exclusive process lock and launches no workers until recovery and preparation finish. Its first local inspection refuses canonical divergence or a missing rollback-safe checkpoint before any provider call. It then attempts one initial Sync: a provider failure may use a still-fresh persisted view, while other failures retain their typed retry/refuse classification. After that attempt, `select_recovery` maps one consistent local inspection as follows: @@ -329,7 +329,7 @@ After that attempt, `select_recovery` maps one consistent local inspection as fo | `EstimatedBatchInDanger(N)` | Retry | Recovery never mutates from an estimate alone. | | `CanonicalDivergence(N)` | Refuse | Standard recovery assumes content identity and is forbidden. | -Closed recovery retains the flush's observed safe block in a local variable. Post-flush Sync must succeed; its provider failure cannot use the initial-sync fallback. The guarded cascade transaction refuses divergence or missing finalized state, requires the persisted safe head to reach the flush observation, and then applies the post-flush policy. It runs even if the refreshed danger verdict is `Safe`: a young unresolved suffix is still doomed after flushing. A crash or retry loses the observation, so another invocation must flush again. +Closed recovery retains the flush's observed safe block in a local variable. Post-flush Sync must succeed; its provider failure cannot use the initial-sync fallback. The guarded cascade transaction refuses divergence or a missing rollback-safe checkpoint, requires the persisted safe head to reach the flush observation, and then applies the post-flush policy. It runs even if the refreshed danger verdict is `Safe`: a young unresolved suffix is still doomed after flushing. A crash or retry loses the observation, so another invocation must flush again. Flush changes only the wallet watermark locally. New divergence can be discovered only by Sync, and the next dispatch or guarded cascade checks it before repair. There is no additional inspection between Flush and Sync. The process lock and task-free startup exclude a competing local writer; revisit this sequencing if startup gains concurrent writers. @@ -386,9 +386,9 @@ Given a trusted checkpoint machine `S` at block `B` (a finalized `dumps//` d 1. **Flush** the wallet nonce (keyed — recovery, unlike plain `setup`, signs) so every previous-instance batch resolves at safe depth `≤ C`, the post-flush safe head. Re-sync `safe_inputs` through `C`. 2. **Fold** (the pure `sequencer-core` engine, shared with the on-chain scheduler so it is consistent by construction): seed the fridge from the `(A, B]` directs (drop batches — already in `S`), replay the `(B, C]` stream, drain the leftover fridge at `C`. Yields `(S', N')` = the advanced app state and the resume nonce. -3. **Fill** a consistent DB: the baseline transaction has already minted a UUIDv4 `EraId` and initialized `RecoveryGeneration = 0`, while leaving the rebuild's `base_executed_input_count` and `base_safe_input_index` NULL. Derive `K = S'.executed_input_count()`. **Anchor the batch tree at `N'`** ([I16](../invariants.md) — the root tip *is* `N'`, no sentinel batch); sequence the `≤ C` inputs so the replay cursor starts past them (they're already in `S'`, while `run`'s first on-chain batch re-drains them by `safe_block`). Capture that root's exclusive safe-input cursor as the durable drain floor, then bind it with `K` in the same transaction that registers `S'` as the initial finalized snapshot at `C`; setup completion requires both non-NULL bases and the snapshot. Later standard recovery uses `max(base_safe_input_index, max valid attribution + 1)`, so invalidating the root cannot re-sequence those inputs. Physical `l2_tx_index` includes unmapped cursor padding and is deliberately distinct from application-history base `K`; the first executable input above the floor is mapped at `K`. `run` boots from this state. +3. **Fill** a consistent DB: write the recovered application dump first, then atomically register its complete `(era, generation = 0, K, C)` history baseline, anchor `N'`, parentless root frame at `C`, snapshot, and `setup_complete`. The collapsed prefix creates no application rows. The first later application input has offset `K`; ordinary recovery falls back to immutable `C` if the root is invalidated. The terminal-drained baseline is a local restore point and is not automatically a canonical comparison checkpoint at `C`. -During recovery the gold frontier (`safe_accepted_batches`) population is **deferred** (`FrontierMode::DeferUntilAnchorSet`): the tree is empty until fill, so simulating acceptance against it would flag every L1 batch as foreign and freeze the frontier ([I15](../invariants.md)). It is populated on `run`'s first sync — once the anchor `N'` is set — so the folded `< N'` history is skipped as trusted collapsed history. `N` is **trusted checkpoint metadata**, not re-verified at recovery time: a wrong-low `N` surfaces at `run` via the content-identity check, but a wrong-high `N` does not — sound because a sequencer-produced finalized dump cannot carry a wrong `N` by construction (see [`cockroach.md`](cockroach.md#data-dictionary) for the full trust boundary). Recovery is a **strict one-shot**: it refuses (terminal) on a DB that is already set up. A retained incomplete DB reuses the still-unexposed era minted by its baseline transaction. Once matching root Tip plus the atomically bound finalized snapshot/`K` exist, that durable fill is authoritative and retry is a no-op; it does not compare stored `K` against a later fold at a newer `C`. A fail-loud partial fill instead requires the operator to wipe and retry, minting another unexposed era. This is not general resume machinery. +During rebuild the accepted frontier is deferred until the baseline exists. The first `run` sync seeds expected nonce `N'` and scans only inputs after `C`, explicitly excluding the trusted prefix. Replaying the old prefix with a later expected nonce could reinterpret a rejected future-nonce batch as accepted. Checkpoint state and nonce remain operator-trusted; the export receipt checks metadata agreement rather than independently verifying the checkpoint. Rebuild is one-shot after completion. File-first creation plus atomic registration removes partial-baseline resume states; a failed transaction leaves only an orphan artifact. See [cockroach recovery](cockroach.md) for the full contract. The detect-and-refuse gate is the *trigger*: a fresh `setup` that finds a previous instance's batches past the checkpoint refuses with exit `40` (`EXIT_SETUP_NEEDS_RECOVERY`), pointing the operator here. @@ -418,6 +418,16 @@ standard recovery on this page**: the cascade reconciles the batch tree's mismatch means canonical state contains executed effects with no reliable local source, so rebuild-from-L1 is the only honest repair. +### Restore points and admission + +Every admitted database retains a rollback-safe application artifact: the +baseline before any local batch is accepted, or an accepted batch snapshot. +Startup may load a newer surviving optimistic snapshot to reduce replay, but +that snapshot alone cannot justify admission because recovery may remove it. +Once an accepted artifact exists, GC may retire baseline bytes while preserving +immutable baseline metadata and active leases. `admission.tla` calls this +`hasRecoveryCheckpoint`; artifact creation and GC remain outside that model. + ## Implementation Constraints These constraints were discovered during TLA+ model checking and are required for correctness: diff --git a/docs/recovery/admission.tla b/docs/recovery/admission.tla index 3f47ff9f..b1a9a667 100644 --- a/docs/recovery/admission.tla +++ b/docs/recovery/admission.tla @@ -13,6 +13,10 @@ * and crash erase them; another attempt must flush again before cascading. * Persisted history and danger facts survive. Terminal-fault telemetry does * not gate admission and is outside the model. + * + * A recovery checkpoint is the baseline or an accepted batch snapshot. It + * survives every standard cascade; an unaccepted snapshot alone is insufficient. + * Artifact creation, leases, and garbage collection are outside this model. *) EXTENDS TLC @@ -48,12 +52,12 @@ MissingSafeHead == "MissingSafeHead" PostFlushViews == {CaughtUp, Behind, MissingSafeHead} VARIABLES controller, admittedRuntime, prepared, flushed, postFlushView, - danger, hasFinalizedSnapshot, hasOpenTip, canonicalDivergence + danger, hasRecoveryCheckpoint, hasOpenTip, canonicalDivergence vars == <> + danger, hasRecoveryCheckpoint, hasOpenTip, canonicalDivergence>> -LocalTerminal == canonicalDivergence \/ ~hasFinalizedSnapshot +LocalTerminal == canonicalDivergence \/ ~hasRecoveryCheckpoint Clean == ~LocalTerminal /\ danger = Safe /\ hasOpenTip Init == @@ -63,7 +67,7 @@ Init == /\ flushed = FALSE /\ postFlushView = NoPostFlushView /\ danger \in DangerStates - /\ hasFinalizedSnapshot \in BOOLEAN + /\ hasRecoveryCheckpoint \in BOOLEAN /\ hasOpenTip \in BOOLEAN /\ canonicalDivergence \in BOOLEAN @@ -73,12 +77,12 @@ Settle == /\ prepared' = FALSE /\ flushed' = FALSE /\ postFlushView' = NoPostFlushView - /\ UNCHANGED <> + /\ UNCHANGED <> MoveTo(next) == /\ controller' = next /\ UNCHANGED <> + danger, hasRecoveryCheckpoint, hasOpenTip, canonicalDivergence>> BeginRun == /\ controller = Idle @@ -103,7 +107,7 @@ SyncCompleted == ELSE /\ controller' = Cascade /\ postFlushView' \in PostFlushViews /\ UNCHANGED <> + hasRecoveryCheckpoint, hasOpenTip>> InitialProviderFailure == /\ controller = InitialSync @@ -123,7 +127,7 @@ FlushCompleted == /\ controller' = PostFlushSync /\ flushed' = TRUE /\ UNCHANGED <> + hasRecoveryCheckpoint, hasOpenTip, canonicalDivergence>> (* Repair methods commit an open Tip in the same transaction as their * mutation. No L1 observation changes, so observed danger cannot reappear; @@ -133,7 +137,7 @@ CommitRepair == /\ hasOpenTip' = TRUE /\ danger' \in {Safe, RetryDanger} /\ UNCHANGED <> + hasRecoveryCheckpoint, canonicalDivergence>> LocalRepairCompleted == /\ controller \in {EnsureOpenTip, RecoverTip} @@ -161,7 +165,7 @@ PrepareCompleted == /\ prepared' = TRUE /\ danger' \in {Safe, RetryDanger} /\ UNCHANGED <> + hasRecoveryCheckpoint, hasOpenTip, canonicalDivergence>> FinalAdmission == /\ controller = FinalCheck @@ -169,7 +173,7 @@ FinalAdmission == THEN /\ controller' = Admitted /\ admittedRuntime' = TRUE /\ UNCHANGED <> + hasRecoveryCheckpoint, hasOpenTip, canonicalDivergence>> ELSE Settle (* Typed I/O/guard failures terminate the attempt. In particular, post-flush @@ -197,7 +201,7 @@ TypeOK == /\ flushed \in BOOLEAN /\ postFlushView \in PostFlushViews \union {NoPostFlushView} /\ danger \in DangerStates - /\ hasFinalizedSnapshot \in BOOLEAN + /\ hasRecoveryCheckpoint \in BOOLEAN /\ hasOpenTip \in BOOLEAN /\ canonicalDivergence \in BOOLEAN diff --git a/docs/recovery/cockroach.md b/docs/recovery/cockroach.md index e4f54550..f02510f0 100644 --- a/docs/recovery/cockroach.md +++ b/docs/recovery/cockroach.md @@ -1,257 +1,127 @@ # Cockroach recovery (`setup --recovery`) -The catastrophe path. When the local DB is lost or has diverged -([`CanonicalDivergence`](../invariants.md#i15-divergence-marker-present--acceptance-frontier-frozen)), -there is no batch tree to cascade — the operator **wipes the data dir and -rebuilds canonical logical state from a trusted checkpoint plus L1**. It is an -operator-driven, one-shot `setup` mode, not a runtime action. There is no -automated database replacement, clone detector, distributed fence, or -partial-fill resume state machine: the supported repair is deliberately the -explicit fresh/wiped-directory flow. +When the local DB is lost or has diverged, the operator rebuilds from a trusted +application checkpoint and L1 in a fresh data directory. This is a one-shot +setup operation. [Standard recovery](README.md) instead keeps the database and +invalidates an unaccepted suffix. -Contrast with **[standard / preemptive recovery](README.md)** (the rest of -`docs/recovery/`): that runs *inside* a live sequencer, uses its own batch tree -to cascade a doomed suffix, and shares the flush machinery. Cockroach recovery -discards the tree entirely and reconstructs `(S', N')` by *folding* L1, then -records the recovered application's absolute executed-input base `K`. - -The pure fold engine ([`sequencer-core/src/scheduler/fold.rs`](../../sequencer-core/src/scheduler/fold.rs)) -is the same scheduler source compiled into the on-chain canonical machine — so -the reconstruction is consistent with L1 *by construction*, not by a parallel -re-implementation. - ---- +The [canonical scheduler fold](../../sequencer-core/src/scheduler/fold.rs) +reconstructs the state. Its terminal drain also prepares the next local frame: +the result is a **resume baseline**, which can be ahead of canonical application +execution at the stopping block. It is not exposed as a finalized comparison +checkpoint merely because the L1 inputs used to construct it are safe. ## Data dictionary -The fold/fill quantities and history metadata below drive the procedure. -Knowing where each is *born* is the key to reading the code. - -| Symbol | Meaning | Where it comes from | +| Symbol | Meaning | Source | |---|---|---| -| **`S`** | The trusted checkpoint machine/app state at block `B`. | Loaded from the dump: `A::from_dump(checkpoint_dump_dir)`. | -| **`A`** | `S`'s last-executed safe block. The fridge is reconstructed from directs in `(A, B]`. | App query: `S.last_executed_safe_block()`. (Persisted in the dump — see `docs/snapshots/format.md`.) | -| **`B`** | The checkpoint's L1 inclusion block. `S` reflects every **batch** with inclusion `≤ B` and **no direct** in `(A, B]`. | Operator arg: `--checkpoint-block`. | -| **`N`** | The checkpoint's resume batch nonce (the scheduler counter at `B`). The bare-metal app *cannot* recompute it, so it rides as checkpoint metadata. | `info.toml`'s `next_batch_nonce` in the dump. | -| **`C`** | The post-flush safe head — the stopping block. The flush resolves every slot the provider remembers at safe depth `≤ C` (best-effort — see step 2). | Return of `flusher.flush_and_wait(...)`. | -| **`N'`** | The resume nonce the new sequencer submits at — `N` advanced by the accepted batches in `(B, C]`. Becomes the **batch-tree anchor**. | Output of `fold_replay(...)`. | -| **`E`, `g`** | The new history era and its recovery generation. `E` is UUIDv4; `g = 0`. | Minted with the baseline schema in one transaction, before external recovery work. | -| **`K`** | The first application-history offset available in `E`: `S'.executed_input_count()`. It is not the replacement DB's physical replay cursor. | Derived after the fold and bound atomically with the initial finalized snapshot row. | -| **`F`** | The exclusive `safe_inputs` cursor already represented by `S'`; standard recovery must never drain below it. | Captured after the recovery root sequences its `≤ C` cursor padding and bound atomically with `K` and the initial finalized snapshot row. | - -**Invariant `A < B`** (checked at load): the checkpoint's executed state must -predate its inclusion block, or the `(A, B]` fridge range is ill-defined. - -`N` and the `(A, B]` batch content are **operator-trusted inputs** — there is no -cheap on-chain oracle to validate the checkpoint against, and recovery -does **not** re-verify them. The only sound independent check would replay the -scheduler's nonce fold from genesis through `C` (a from-genesis frontier is the -one quantity independent of the trusted `N`) — i.e. re-fetch and reprocess all of -L1, which recovery deliberately does not do. - -This is sound because the checkpoint is a **sequencer-produced finalized dump**, -where the tuple is self-consistent by construction: `info.toml`'s -`next_batch_nonce` is the closing batch's nonce + 1, and a dump only reaches -*finalized* once that batch is promoted (observed accepted on L1), so `N` is -exactly the scheduler's position at `B`. A wrong `N` can therefore arise only -from a corrupted or externally-produced checkpoint — already outside the trust -boundary, the same one that accepts `S` with no verifier. - -If that boundary is ever violated, the two wrong-`N` shapes are **not** -symmetric: -- a wrong-**low** `N` (or a mis-stated `(A, B]`) surfaces loudly at `run` — the - rebuilt root collides with the still-live L1 batches in `[N', M)` and trips the - content-identity check (I15); -- a wrong-**high** `N` is **not** caught — the `[M, N)` history was never reached - on-chain, so nothing collides; the off-chain frontier accepts the local batch - against itself while the real scheduler ignores it. This is why the checkpoint - must be a trustworthy finalized dump, not merely "some app bytes". - -This is a concrete boundary of the content-identity check: it is complete for at/above-anchor accepted -batch content identity, not checkpoint/application correctness or arbitrary -canonical divergence. Absence of `canonical_divergence` is not a proof that a -checkpoint outside the trust boundary was valid. - ---- +| `S` | Trusted application state at checkpoint block `B`. | Restored application dump. | +| `A` | Last executed application safe block in `S`; pending directs are seeded from `(A, B]`. | Application progress in the dump. | +| `B` | Checkpoint inclusion block. | Exported `checkpoint.toml`; must equal the configured checkpoint block. | +| `N` | Scheduler's next batch nonce at `B`. | Exported receipt, checked against immutable `info.toml`. | +| `C` | Post-flush safe stopping block. | Flusher result. | +| `N'` | Next batch nonce after folding through `C`; the new batch-tree anchor. | Fold result. | +| `K` | Application count after the terminal drain; first local history entry is `K`. | Recovered application progress. | +| `E`, `g` | New UUIDv4 era and generation zero. | Atomic completed-baseline registration. | + +The checkpoint contract requires `A < B`, checked at load. The sole exception +is the known empty genesis checkpoint (`B = 0`, next nonce and app count zero). +At a non-genesis `A = B`, a direct arriving after the accepted batch in block +`B` can still be pending; the empty `(A, B]` seed would silently omit it. +A recovery export carries a canonical application dump and a separate receipt; +baseline downloads and ordinary optimistic snapshots have no such receipt. + +### Trusted checkpoint boundary + +The application state, resume nonce, and relationship between the checkpoint and +L1 are operator-trusted. The receipt catches accidentally mixing an artifact, +nonce, or configured inclusion block; it does not independently verify state +against L1. A wrong checkpoint nonce, whether low or high, is outside the +supported model. The content-identity check verifies newly observed acceptance +after the baseline, not the opaque prefix or checkpoint correctness. + +An independent verification would need a trusted canonical-machine checkpoint +or replay from an independently trusted origin. The infrastructure subscriber's +application dump is not a substitute for that watchdog trust boundary. ## The procedure: flush → fold → fill -Opening the fresh replacement DB first commits one baseline transaction: the -schema and a UUIDv4 era with generation zero. Because neither the folded -application nor the recovery-root -cursor exists yet, `base_executed_input_count` and `base_safe_input_index` start -NULL. That era remains externally unexposed until setup completes. +1. **Load the checkpoint.** Restore `S`, read both metadata files, verify their + nonce agreement and the configured `B`, then derive `A` and require `A < B` + or the known empty genesis checkpoint. +2. **Flush stranded transactions.** Consume unresolved wallet nonce slots and + wait for safe finality, obtaining `C`. The lost database cannot supply its + previous watermark, so the flush uses the provider's pool view. A dropped + transaction alive elsewhere can evade that view; a later accepted foreign or + mismatched landing after `C` freezes the new instance and requires another + rebuild. The trusted provider is fail-stop, not Byzantine. +3. **Re-sync raw L1 inputs.** The safe head `H1` must cover `C`; it can be later. + Acceptance projection is deferred while the new local tree is absent. +4. **Source disjoint fold ranges.** Seed external directs in `(A, B]`, then + replay all raw inputs in `(B, C]`. Sender classification excludes own batch + envelopes from the direct-input seed queue. +5. **Fold and drain.** The scheduler processes the stream with expected nonce + `N`, then drains every remaining direct through `C`, producing `(S', N')`. + A young direct still waiting in the canonical scheduler may therefore already + be present in `S'`. The resumed frame covers it before executing new user ops. +6. **Write the baseline artifact, then publish it.** First create and durably + sync the immutable dump. One SQLite transaction then creates history + `(E, 0, K, C)`, sets anchor `N'`, opens its parentless root frame at `C`, + registers the baseline artifact, and records `setup_complete`. It creates no + application-input rows for the collapsed prefix. + +On the first `run` sync, acceptance starts at `N'` and scans only raw inputs +whose block is **strictly greater than `C`**. Nonce filtering alone is unsound: +a previously rejected future-nonce batch inside the old prefix could match the +new expected nonce. The opaque prefix is never classified again. + +Inputs in `(C, H1]` remain available to the inclusion lane. Its next complete +reconciliation executes them once and records application entries beginning at +`K`. Raw L1 input indices and application offsets remain separate coordinates. + +## Recovery and retention + +`C` is the immutable fallback reconciliation boundary. While valid frames +survive, their latest `safe_block` gives the already-reconciled boundary. If +standard recovery invalidates the original root, it falls back to `C`, so +inputs represented by the baseline are never executed again. Canonical +application rows belonging to invalidated batches are deleted atomically with +the generation change and suffix invalidation. + +Startup loads the latest surviving batch-close snapshot, falling back to the +baseline. Admission requires a **rollback-safe checkpoint**: either that +baseline or a retained accepted batch snapshot. An optimistic snapshot alone +cannot satisfy this requirement because a cascade may discard its whole suffix. + +Once an accepted post-baseline batch snapshot exists, standard recovery cannot +invalidate it or return to the original baseline. GC can retire the baseline +artifact, subject to download leases. Immutable baseline metadata remains. +Snapshots with equal application counts remain distinct artifacts associated +with distinct batches; acceptance and retention never infer identity from count. + +## Crash-safety & idempotency -``` - ┌─ load S, derive A & N, require A < B - checkpoint │ - (S @ B, N)│ flush wallet nonce ───────────────► C (post-flush safe head) - │ │ - │ ▼ - L1 ────────┼──► re-sync safe_inputs through C (frontier population OFF) - │ │ - │ ▼ - │ source seeds = (A,B] directs (drop batches: in S) - │ replay = (B,C] stream - │ │ - │ ▼ - │ fold_replay(S, N, seeds, replay, C) ──► (S', N') - │ │ - ▼ ▼ - fill: anchor = N', root tip @ N', finalized snapshot of S', - replay cursor past the ≤C directs ──► run boots here -``` +A completed rebuild refuses another `setup --recovery`. Before completion, +there is no partially registered history or recovery root to resume: -1. **Load `S`; derive `A`, `N`; require `A < B`.** Read the dump - (`from_dump` + `info.toml`); `A = S.last_executed_safe_block()`. -2. **Flush → `C`.** Settle the wallet nonce (keyed L1 no-ops; this is where - cockroach recovery composes with the standard flush). `C` is the stopping - point: directs beyond `C` are `run`'s job, not the fold's. - **This flush is best-effort by construction:** the wiped DB carries no - wallet-nonce watermark, so the durable-anchor half of the completion test - is vacuous — the flush resolves only the slots the provider remembers, - and a zombie tx the local node forgot but the network still holds is - unresolvable here (plain `setup`'s detection gate shares the same false - negative). The content-identity check is what makes this acceptable: such - a zombie landing at/above `N'` is detected and freezes the frontier - instead of silently diverging, and the repair is another wipe-and-rerun — - cockroach recovery recovers from the failure of its own flush. If ever - needed, an operator-supplied flush floor taken from the old DB's - watermark is a sound option: the value is fail-safe under corruption - (too high wastes a few no-ops; too low degrades to exactly best-effort), - so reading it from an untrusted half-destroyed DB does not violate the - don't-trust-local-state premise. -3. **Re-sync `safe_inputs`; flush-view coherence.** The reader syncs to the *live* safe - head `H1` (normally `> C` — real time passed while the flush awaited safe - finality); refuse only if it *lags* `C` (a load-balanced RPC replica could - serve a stale view). So `safe_inputs` ends up holding directs through `H1`, - not just `C` — step 6 is careful to drain only the `≤ C` ones. **Gold - frontier population is OFF for recovery's syncs** — the tree is empty until - step 6, so populating the frontier would mark every L1 batch `Foreign` and - falsely freeze it. The frontier is deferred to `run`'s first sync. -4. **Source the fold inputs.** Seeds = the `(A, B]` **directs** (drop - `sender == batch_submitter` — those are batches, already in `S`); replay = - the full `(B, C]` stream. The seed filter *is* the scheduler's own - sender-based classification. -5. **Fold `(S, N)` → `(S', N')`.** The engine seeds the fridge from `(A, B]`, - replays `(B, C]` (force-executing overdue directs, applying accepted - batches, draining covered fridge directs), drains the leftover fridge at `C`, - and advances the nonce to `N'`. -6. **Fill the DB.** Derive `K = S'.executed_input_count()`. **Anchor the batch tree - at `N'`** (the root tip *is* `N'` — there is no sentinel batch); open the root - tip at frame `safe_block = C` and sequence **only the `≤ C` safe inputs** so - the replay cursor starts past them. The drain is sender-unfiltered — it - includes the `≤ C` batch-submitter rows alongside the user directs the fold - folded into `S'`, exactly as the genesis tip drains its whole span; those - rows are sequenced (cursor padding), never executed, so the - `sender != batch_submitter` *seed* filter does not reappear here. Capture the - root's exclusive safe-input cursor as `F`, then snapshot `S'` as finalized at - `C` and bind `(K, F)` in the same SQLite transaction; setup completion - refuses until both bases and the finalized snapshot exist. The `(C, H1]` - directs the resync pulled in past `C` stay **undrained** — `run`'s lane leads - and executes them exactly once as the safe frontier advances `C → H1`. - (Draining them here instead would skip them on catch-up while `S'` never - executed them — a vanished deposit / divergence; this is why the fill uses a - `≤ C`-capped `open_recovery_tip`, not the generic whole-table drain.) `run` - boots from this state. Those padding rows advance physical `l2_tx_index`, - but not `K` and receive no `executed_inputs` mapping: they are physical - cursor attribution for inputs already reflected in `S'`, not newly executed - application history. The first executable direct above `F` receives logical - offset `K`; applying it moves the recovered application to `K + 1`. +- A failure during artifact creation leaves setup incomplete. +- A failed registration transaction leaves neither baseline history, root, + anchor update, snapshot row, nor completion marker; any durable file is an + orphan for cleanup. +- A successful transaction establishes all those facts together. There are no + nullable baseline coordinates and no physical replay padding. ---- +Early identity pinning and raw L1 ingestion can survive an incomplete attempt. +They do not establish an application-history era. The process lock and setup +admission exclude runtime serving before the complete baseline exists. ## Code map -| Step | Code | +| Responsibility | Code | |---|---| -| entry / branch | [`setup()`](../../sequencer/src/commands/setup/mod.rs) branches on `config.recovery` after the shared prefix (identity pin + initial sync) | -| 1. load + `A < B` | [`recover()` step 1](../../sequencer/src/commands/setup/mod.rs) — `from_dump`, `read_info`, `CheckpointNotBeforeBlock` | -| 2. flush → `C` | `recover()` step 2 — `MempoolFlusher::flush_and_wait` (see [`recovery/flusher.rs`](../../sequencer/src/recovery/flusher.rs)) | -| 3. re-sync + coherence | `recover()` step 3 — `set_frontier_mode(DeferUntilAnchorSet)` + `sync_to_current_safe_head` + `ResyncBehindFlushView` | -| 4. source seeds/replay | `recover()` step 4 — `Storage::safe_inputs_in_block_range` + the `sender != submitter` filter + `to_fold_input` | -| 5. fold | [`fold_replay`](../../sequencer-core/src/scheduler/fold.rs) | -| baseline history | [`baseline_migration`](../../sequencer/src/storage/open.rs) — UUIDv4 era + generation zero + NULL rebuild base in one baseline transaction | -| 6. fill | [`fill_recovery_state`](../../sequencer/src/commands/setup/fill.rs) — anchor + `open_recovery_tip` (`≤ C`-capped drain at frame `safe_block = C`) + atomic finalized-snapshot/`K` bind | -| anchor mechanism | [`trg_enforce_nonce_contiguity`](../../sequencer/src/storage/migrations/0001_schema.sql) + `compute_next_nonce` + the anchor-aware frontier in [`safe_accepted_batches.rs`](../../sequencer/src/storage/safe_accepted_batches.rs) | - ---- - -## Load-bearing constraints & invariants - -- **`A < B`** — checked at load ([`SetupRecoveryError::CheckpointNotBeforeBlock`]). -- **Disjoint fold ranges** — seeds `(A, B]`, replay `(B, C]`, strictly disjoint; - the fold's always-on asserts enforce ascending order *within* each and a - strict block boundary *between* (directs at block `B` are seeds; the replay - starts strictly after). -- **Frontier deferral** — recovery's syncs never populate the gold frontier (the - tree is empty); `run`'s first sync populates it once `anchor = N'` is set, so - the folded `< N'` batches are skipped as trusted collapsed history rather than - flagged `Foreign`. See the anchor-aware-frontier note on - [I15](../invariants.md#i15-divergence-marker-present--acceptance-frontier-frozen). -- **Anchored root** — the rebuilt tree has exactly one valid parentless root, - carrying `N'` ([I16](../invariants.md#i16-the-batch-tree-has-exactly-one-valid-parentless-root-carrying-the-deployments-anchor-nonce)). -- **No double-execution, no lost directs** — the `≤ C` directs are sequenced - (cursor advances) but the finalized snapshot's `l2_tx_index` is set *after* - sequencing, so `run`'s catch-up (`offset > l2_tx_index`) skips them; they are - already in `S'`. Symmetrically, the `(C, H1]` directs are **not** sequenced - here (`open_recovery_tip` caps the drain at `C`), so the cursor sits below them - and `run`'s lane leads + executes them exactly once — neither double-executed - nor lost. The durable `base_safe_input_index = F` also survives invalidation - of the recovery root: subsequent standard recovery derives its drain cursor - as `max(F, max valid attribution + 1)` and cannot re-sequence or re-execute - the `≤ C` prefix after the root's padding leaves the valid view. -- **History base is application state, not cursor padding** — rebuild baseline - leaves `(K, F)` NULL; fill derives `K` from `S'.executed_input_count()` and - `F` from the root's exclusive safe-input cursor, then binds both with the - initial finalized snapshot. Setup requires all three before completion. Physical - `l2_tx_index` remains a rowid replay cursor and may be greater because it also - covers sequenced-but-not-executed batch-envelope padding. Those padding rows - are deliberately absent from the canonical mapping; new executable history - is attributed contiguously from `K`. - ---- - -## Crash-safety & idempotency - -`setup --recovery` is a **strict one-shot** on a freshly-wiped DB: - -- It **refuses** (terminal, exit 30) if `setup_complete` already exists — the - model is "delete the data dir and re-run", not resume-a-live-deployment. -- The `setup_complete` marker is the linearization point, written **last**; it - requires non-NULL `K`, non-NULL `F`, and the finalized snapshot. -- A crash *before* the marker is handled fail-loud, not by blind resume: - - Retaining an incomplete DB retains the UUIDv4 era minted by its baseline - transaction. An early retry therefore reuses that still-unexposed era; it - does not rotate merely because the command restarted. - - A **completed** fill (finalized snapshot present — the last write) re-runs as - a safe no-op once the root Tip's `N'` matches. Its atomically bound - finalized snapshot/`(K, F)` tuple is authoritative; retry never compares - that stored base with a later fold at a newer `C`. - - A **same-`N'`** re-run of a fill that crashed *mid-fill* (root tip exists, no - finalized snapshot) is **refused** (`PartialRecoveryIncomplete`). It is *not* - idempotent: a re-sync may have advanced `C` with new directs (which leave - `N'` unchanged) that resuming would leave unsequenced, leaving the snapshot - cursor behind `S'` and double-draining them on `run`. Wipe and re-run. - - A **different-`N'`** re-run (a different checkpoint, or the same one after - `C` advanced with new accepted *batches*) is **refused** - (`PartialRecoveryMismatch`): the durable root tip carries the old nonce and - cannot be silently re-anchored. Wipe and re-run. - - `setup --recovery` over **foreign residue** — a finalized snapshot with no - root tip, left by a crashed plain `setup` (which writes the genesis snapshot - before its marker) — is **refused** (`RecoveryOverResidualSnapshot`). A - completed cockroach fill always has both a snapshot and a root tip; keeping - the old snapshot would mark setup complete over genesis instead of `(S', N')`. - Wipe and re-run. - - A subsequent **plain `setup`** over recovery residue is **refused** - (`GenesisOverRecoveryResidue`, anchor `≠ 0`) — it must not root genesis at - the recovery nonce. - -Any fail-loud partial-fill refusal requires the explicit operator wipe/retry. -That fresh baseline necessarily mints another era, while the discarded era was -never exposed by a completed setup. This narrow completed-fill no-op is not a -general resumable rebuild protocol. - -(See the deep-review remediations: these guards close the partial-recovery -revalidation gap, including the same-`N'`/advanced-`C` double-drain — external -review 2026-06.) +| Load, flush, sync, fold | [`commands/setup/mod.rs`](../../sequencer/src/commands/setup/mod.rs) | +| Durable baseline artifact | [`commands/setup/fill.rs`](../../sequencer/src/commands/setup/fill.rs) | +| Atomic baseline completion | [`storage/lifecycle.rs`](../../sequencer/src/storage/lifecycle.rs) | +| Scheduler fold | [`scheduler/fold.rs`](../../sequencer-core/src/scheduler/fold.rs) | +| Accepted-prefix boundary | [`storage/safe_accepted_batches.rs`](../../sequencer/src/storage/safe_accepted_batches.rs) | +| Snapshot selection and GC | [`storage/snapshot_dumps.rs`](../../sequencer/src/storage/snapshot_dumps.rs) | diff --git a/docs/review/register.md b/docs/review/register.md index 815c0409..dc318a90 100644 --- a/docs/review/register.md +++ b/docs/review/register.md @@ -48,16 +48,12 @@ remaining dated ledgers stay valid. "application internal error"; the reason stays on the lane error and the log. 6. **WS session hygiene** — a mid-session transient read error tears down - with no close frame; a beyond-head `from_offset` idles forever (currently - e2e-pinned as intended). The accepted - [Track 3 contract](../plans/2026-07-track3-feed-replay-design.md#51-admission-and-continuity) - chooses a typed ahead-of-head error; API implementation and re-pinning remain open. -7. **WS invalidation/rollback contract** — `/ws/subscribe` still pages by - physical rowid with no `HistoryVersion` claim, so a cursor-resumed - subscriber silently keeps invalidated rows across recovery. Interim - consumer rule: treat any socket drop as a potential discontinuity. - Closure is exclusively owned by the - [Track 3 handoff](../plans/2026-07-track3-feed-replay-design.md#7-ordered-implementation-handoff). + without a close frame. Ahead-of-head admission is closed (2026-09-16): a + typed HTTP409 refuses it before upgrade. +7. **Closed** (2026-09-16): mandatory era/generation/application-count claims + reject resume across recovery; snapshot headers provide cold-bootstrap + coordinates. Current application suffix replacement is atomic with the + generation bump. See the [Track 3 contract](../plans/2026-07-track3-feed-replay-design.md). 8. **Fee-determinism contract under-specified** — the LSB-first floor-after-each-multiply order is implemented but not stated as contract (`sequencer-core/src/fee.rs`). Load-bearing for the C++ scheduler port; @@ -67,11 +63,11 @@ remaining dated ledgers stay valid. 9. **`trg_enforce_nonce_contiguity` NULL hole** — a dangling parent makes the comparison NULL and the trigger silent; mitigated by `foreign_keys=ON` on every writer connection, but the trigger itself is not NULL-safe. -10. **`seal_and_open_next_batch` takes an unchecked `next_safe_block`** - (assert equality with the head or drop the parameter). (The bare - `close_frame_and_batch` is `#[cfg(test)]` as of 2026-09-03.) -11. **Write-only columns** `safe_accepted_batches.{first_frame_safe_block, - inclusion_block}` have no production reader — drop or mark audit-only. +10. **Closed** (2026-09-16): batch sealing asserts that the next frame retains + the durable Tip clock; complete L1 reconciliation owns clock advancement. +11. **Partially closed** (2026-09-16): `safe_accepted_batches.inclusion_block` + drives accepted snapshot selection and export. `first_frame_safe_block` + remains audit-only and may be removed in a separate cleanup. 12. **`direct_q` is unbounded in the shared scheduler** — an adversarial deposit flood is bounded in time (force-drain) but not bytes; a per-input cap or byte budget closes a (very expensive) guest-OOM vector. @@ -260,11 +256,9 @@ Statuses swept 2026-08-22 and updated through 2026-09-04. insufficient-balance silent no-op and replay-determinism pins; the young-never-submitted-batch cascade-policy pin; the `recover_aging_tip` torn/no-Tip entry; the cascade-with-backward-clock pin. -- **The batch-close failure half of I7**: pre-insert a `dumps` row with a - colliding prefix so the seal transaction fails on UNIQUE, and assert the - batch stays the open Tip. Companion state variant: delete the directory - under a DB-referenced snapshot row and assert the loud terminal shape - (the WAL-rewind *cause* stays unsimulable). +- **Closed** (2026-09-16): I7's colliding-artifact test asserts that failed + snapshot registration rolls back the seal, successor Tip, and cached head. + Snapshot endpoint tests cover referenced artifact deletion as a terminal fault. - **Harness levers to build with their tests**: pending-tx capture + re-inject (`txpool_content`/raw-tx before `drop_all_pending_txs`, then `eth_sendRawTransaction`) → unlocks the zombie e2e, the headline @@ -291,6 +285,15 @@ Statuses swept 2026-08-22 and updated through 2026-09-04. Each entry: the decision, its reason, and where the reasoning now lives. +- **Application-only current history** (2026-09-16): retain every raw L1 input + and original batch/frame/user-op record, but replace the invalidated flattened + application suffix. The recovered prefix is opaque. Mandatory offsets and + versioned claims replace the mixed replay log and sparse mapping. Acceptance + facts select immutable per-batch snapshots without promotion or restamping; + per-batch cadence and end-of-block watchdog comparison remain. The complete + model lives in [application history](../plans/application-history.md), I5–I11, + I18/I20, and the snapshot lifecycle. + - **No architectural restructure** (2026-06-10): one file per writer role, `*_in(tx)` free functions composing into larger transactions, storage-owns-SQLite / lane-owns-filesystem — the layout is sound and is @@ -376,7 +379,7 @@ Each entry: the decision, its reason, and where the reasoning now lives. state-file check would violate that lifecycle contract → snapshot handlers, command error classification, and genesis harness tests. - **Execution-offset continuity has one enforcement point** (2026-09-07): - the SQLite trigger rejects a noncanonical offset inside the physical-row + the SQLite trigger rejects a noncanonical offset inside the application-input transaction. The duplicate Rust loop was removed; rollback, invalidation, and offset-reuse tests remain → I20. - **Module homing** (2026-08-19): command brackets in `commands/` (with @@ -667,7 +670,7 @@ these codes; their concepts now live here: | R3 | `synchronous=FULL` decision | `storage/open.rs` | | R4 | exit-code contract | `commands/error.rs`, runbook | | R5 | fail-loud check policy | invariants check policy | -| F1–F10 | 2026-06 correctness findings | settled above; F7 = the open "WS invalidation/rollback contract" finding | +| F1–F10 | 2026-06 correctness findings | settled above; F7 = the closed "WS invalidation/rollback contract" finding | | I1–I20 | invariants (stable, still in use) | `docs/invariants.md` | | D1–D11, H1–H14, S-A, P1–P8 | 2026-08-18 defects / harvest / structural fix / premise items | settled above + ADR | | WP1–WP11 | 2026-06 work packages (all landed) | settled above | @@ -702,3 +705,4 @@ for `2026-06-10-correctness-review.md`, `2026-06-10-simplification.md`, | 2026-09-07 | PR #28 premise review and maintainer-approved simplification | Ordered recovery replaces the phase driver; diagnosed terminal runtime faults abort immediately; ordinary shutdown and snapshot leases remain; reader drain race and duplicate offset check fixed | Current ADR and recovery design; finding 33 and settled decisions above. Validation: 692 host tests, seven targeted restart/outage E2Es, workspace check, strict Clippy, formatting, and admission TLC passed. The broader stale-batch recovery E2E reached its watchdog comparison but was blocked by the host Lua emulator 0.21 loading the pinned 0.20 image (archive version mismatch); no protocol pin was changed. | | 2026-09-09 | Application, inclusion lane, and public DEX integration branch | Native progress ownership, typed validation failures, mutable independent checkpoints, optional canonical inspection, and lane bookkeeping simplified; ingress CORS and Lua 5.4 parity restored. Reference C bridge port kept separate. | [Application/lane review](2026-09-09-application-lane-dex-review.md); current Application and snapshot contracts. Workspace check, strict Clippy, 697 host tests, and 62 watchdog tests passed; private DEX conformance remains unverified. | | 2026-09-11 | Reference C bridge port and review boundary | Keep the current Application contract, runtime payload bound, paired progress, filesystem-owned checkpoint disposal, and host failure fixes together. Engine-dependent API refinements and external-engine conformance remain follow-ups. | Settled decisions and owed tests above; [C binding guide](../protocol/c-application-binding.md), Application contract, and snapshot lifecycle. | +| 2026-09-16 | Application-history and Track 3 implementation, with independent storage/recovery/snapshot review | Replaced mixed replay and sparse attribution with current application inputs; complete atomic baselines; acceptance-derived immutable snapshots; HTTP restore/recovery archives and mandatory WS claims. Review narrowed equal-block recovery to empty genesis to avoid losing same-block pending directs. | [History design](../plans/application-history.md), current invariants/API/snapshot/recovery docs. Validation: 693 workspace tests, strict Clippy, formatting, 62 watchdog tests, admission TLC (155 distinct states), and the Anvil recovery/old-claim refusal gate passed. Full canonical watchdog comparison remains blocked by the local emulator 0.21 vs repository 0.20 environment; no pin changed. | diff --git a/docs/snapshots/format.md b/docs/snapshots/format.md index cdf04eb9..963a5323 100644 --- a/docs/snapshots/format.md +++ b/docs/snapshots/format.md @@ -15,7 +15,7 @@ This document covers two things: the [Application contract](../protocol/application-contract.md#6-checkpoint-lifecycle). It does NOT define when snapshots are triggered, how the inclusion lane -records and promotes them, how the HTTP layer serves them, or recovery +records and selects them, how the HTTP layer serves them, or recovery interactions. Those are layered above the trait and live in their own modules. @@ -178,7 +178,7 @@ format itself does not provide one. This document deliberately does not define: - When the inclusion lane decides to take a snapshot. -- How dumps are registered, promoted from pending to finalized, or +- How dumps are registered, selected by acceptance, or garbage-collected. - The on-the-wire archive format for streaming a dump over HTTP. - Inspect-state procedures on other implementations (Cartesi Machine, diff --git a/docs/snapshots/lifecycle.md b/docs/snapshots/lifecycle.md index 2182b349..f99ad0d3 100644 --- a/docs/snapshots/lifecycle.md +++ b/docs/snapshots/lifecycle.md @@ -1,488 +1,132 @@ -# Snapshot Lifecycle +# Snapshot lifecycle -How the inclusion lane takes, promotes, serves, and garbage-collects application -snapshots — and **why** it is built the way it is. This is the companion to -[`format.md`](format.md): that doc defines the on-disk *format* (the -`Application` dump trait and the wallet's wire encoding) and explicitly scopes -the lifecycle out; this doc owns the lifecycle. +Snapshots preserve application state at a declared execution boundary. Their +contents and boundary are immutable. L1 acceptance is a separate durable fact: +there is no promotion operation and no mutable finalized pointer. -Audience: anyone changing the snapshot path, the inclusion lane's -safe-frontier processing, or recovery. Read [`AGENTS.md`](../../AGENTS.md) first -for the sequencer/scheduler duality and the optimistic-confirmation model. +## Artifact and boundary -> **Code is the source of truth.** This doc explains *why*; the symbol names -> below (functions, tables, columns) can drift. Verify them against the current -> code before relying on them. - -## Invariants & landmines - -Read these first — the load-bearing rules, and the things that look wrong but -aren't. Section references point to the full reasoning below. - -**Invariants** (hold at all times): - -- **Always-load.** A finalized snapshot exists before the lane starts (genesis - at cold start). Absence is a bug, surfaced fail-loud as - `CatchUpError::NoSnapshot` — never a branch the happy path handles. (§2) -- **Tip exists before the lane.** A valid open Tip exists when the lane starts: - the guarded `ensure_open_tip_for_recovery` operation opens the genesis Tip on a fresh - DB (after the initial safe-head sync, before the lane); recovery reopens it - atomically across cascades. The lane - loads the resulting head from storage (fail-loud if absent), so it only ever - *loads* — it never branches on tip existence or initializes one. (§7) -- **A committed promotion implies an advanced drain.** Promotion is folded into - the drain's attributed transaction - (`close_frame_only_with_executions`), so promotion, physical - drain, and logical mappings commit together—this is what makes a crash safe. - (§5, §6) -- **No dangling row.** No `dumps` row references a missing directory: create the - file before the row; delete the row before the file. (§7) -- **One resume checkpoint.** The same row supplies the `from_dump` prefix, - physical replay cursor, and canonical executed-input count. Startup checks - the loaded app count before replay, so state and either coordinate cannot - drift. (§4) -- **Snapshot `l2_tx_index` is the global valid replay head**, not the batch's - own last offset — so an empty batch doesn't reset catch-up to genesis. (§3) -- **Snapshot `executed_input_count` is storage-derived `H`.** Registration - fails loud if the application's count differs from the canonical mapping; - promotion carries the count with the physical cursor. (§3–5) -- **Storage is SQLite-only; the lane owns FS cleanup.** That boundary *is* the - GC crash-ordering guarantee — don't push filesystem work into - `storage/snapshot_dumps.rs`. (§7) - -**Landmines** (deliberate, or foot-guns): - -- **Per-range promotion intentionally skips intermediate blocks** — only the - range's max nonce is promoted. Sound because nonces land monotonically and the - skipped checkpoints were never observable. Not a bug. (§5) -- **`Storage::promote_finalized` (standalone) is `#[cfg(test)] pub(crate)`.** - Promoting outside the drain transaction re-opens the wedge (§6); the helper - exists only in test builds. Production promotes via - `close_frame_only_with_executions`. -- **Several snapshot `Storage` methods are `#[cfg(test)]`** — non-atomic - siblings of the atomic production methods (`gc_dump_rows` vs - `gc_unreferenced_dumps`; `acquire_dump_lease` vs `acquire_*_lease`; - `clear_pending_dumps` vs `clear_pending_dumps_in`). Use the atomic ones in - production. -- **GC runs after a promotion, not on a timer.** Don't move it back to the idle - path — it starves under load. (§7) -- **A *missed* promotion on crash is fine; a *re-promotion* is the wedge.** That - asymmetry is the whole point of §6 — don't "optimize" by promoting eagerly or - separately from the drain. -- **The HTTP lease is held by a drop-guard inside the response body, not - released after the stream.** A linear acquire → stream → release would *leak* - the lease on client disconnect — code after the `.await` doesn't run when the - body future is cancelled. (§7) - -## 1. Purpose & model - -A snapshot is a durable copy of the application's canonical state at a known -physical replay cursor and canonical executed-input boundary. It exists for -three consumers: - -- **Catch-up** (lane startup): instead of replaying the entire L2-tx history, - the lane loads the freshest snapshot and replays only the tail after it — a - single *load-then-replay* path. -- **The watchdog** (operator): polls the **finalized** snapshot to verify the - sequencer's state against an independent canonical machine advanced through - L1. -- **Indexers** (operator): fetch the **latest** snapshot, then subscribe to the - L2-tx feed from that snapshot's offset. The current API exposes the physical - `l2_tx_index`; Track 3 will expose/admit the canonical - `executed_input_count` with `HistoryVersion`. - -Three SQLite tables back it (`storage/migrations/0001_schema.sql`): - -| Table | Holds | -|----------------------|----------------------------------------------------| -| `dumps` | `(id, prefix, lease_count)` — one row per on-disk dump directory | -| `pending_snapshots` | `(nonce, dump_id, l2_tx_index, executed_input_count)` — snapshots of closed-but-not-yet-L1-confirmed batches | -| `finalized_snapshot` | single row `(dump_id, inclusion_block, l2_tx_index, executed_input_count)` — the latest L1-confirmed state | - -`prefix` is the **dump directory** — a structured dir the sequencer owns -(`ingress/inclusion_lane/dump_info.rs`): +Every artifact is a sequencer-owned directory: ```text dumps// - state app-owned file or directory — the prefix handed to - Application::{create_dump, from_dump} - (opaque to the sequencer; see format.md) - info.toml sequencer-owned checkpoint metadata: - format_version, next_batch_nonce (N), l2_tx_index, - promoted_inclusion_block (B) + info.toml format_version, next_batch_nonce + state opaque app-owned file or directory ``` -`info.toml` makes a finalized dump a **self-contained checkpoint** for the -recovery handoff: `N` and the replay cursor are known at batch -close and written then; `B` is known at promotion and stamped **in place** -afterwards (and re-stamped from the authoritative DB row at every startup, -closing the commit-then-stamp crash window). An in-place update of a file -*inside* the dir changes no path, so the no-dangling-row invariant, leases, -and GC — all keyed on the immutable directory path — are untouched. The dir -name itself stays opaque; metadata lives only in `info.toml`. - -`executed_input_count` is intentionally not another `info.toml` field. It is -already canonical application state inside `state`, while SQLite stores the -independent expected value used to reject a mismatched dump at startup. The -physical replay cursor remains sequencer-owned checkpoint metadata. - -The split between **pending** and **finalized** mirrors the sequencer's -optimism: a batch closes off-chain (soft) → its snapshot is *pending*; the -batch lands safe on L1 → its snapshot is *promoted* to finalized. - -Egress lease acquisition captures `HistoryVersion` in the same transaction as -the selected artifact and its canonical count. The returned metadata remains -associated with those bytes even if another snapshot is promoted afterward. -HTTP projection of this identity belongs to the Track 3 consumer cutover. - -The storage half lives in `storage/snapshot_dumps.rs` (SQLite only — no -filesystem); the lane half in `ingress/inclusion_lane/snapshot.rs` + -`dump_info.rs` (drives the trait and FS work). That split is load-bearing for -the GC crash-ordering (§7). - -## 2. The always-load invariant - -**A finalized snapshot always exists by the time the lane starts.** The runtime -establishes it across the setup/run boundary: `setup` writes and registers the -genesis dump directly as finalized (bypassing pending) before atomically -committing setup completion. On every `run`, startup recovery refuses a -missing finalized-snapshot fact before any provider call, and task-free -`PreparedRuntime::prepare` requires and re-stamps the referenced artifact before -runtime admission over durable facts. This gives catch-up a single unconditional path — -there is always *something* to load — and turns "no snapshot" into a violated -invariant surfaced fail-loud as `CatchUpError::NoSnapshot`, never a branch the -happy path handles. -The same applies when the durable row exists but its referenced metadata or app -artifact is missing or structurally corrupt: startup classifies that provenance -as terminal instead of restart-looping. Other filesystem availability errors -remain operational. - -## 3. Taking a snapshot at batch close - -When the lane closes a batch (`close_batch_with_snapshot`), ordering is chosen -for crash/error safety: - -1. **The dump directory first, outside any transaction** - (`dump_info::create_dump_dir_with_info`): the dir, its `info.toml` - (`next_batch_nonce` = closing nonce + 1, the replay head; `B` left for - promotion), then the `Application`'s dump under `state` — all written and - `fsync`ed. On failure nothing is sealed — the batch stays the open Tip; the - error propagates per the lane's fail-loud policy (the process exits, and the - retry happens on the next boot after catch-up). -2. **One transaction seals the batch, opens the next, and inserts the - `pending_snapshots` row** (`close_frame_and_batch_with_pending_dump`). A - committed close therefore *always* has a promotable pending row; a tx failure - rolls the seal back, leaving only an orphan directory (reaped by the startup - sweep, §7). - -This atomicity closes a "seal succeeds, snapshot insert fails, promotion later -wedges forever on `QueryReturnedNoRows`" gap (commit `0a98cf9`). - -The pending row records `l2_tx_index` = the **global valid replay head** -(`valid_ordered_l2_tx_head`, `MAX(offset)` over `valid_sequenced_l2_txs`), *not* -the batch's own last offset. An empty batch (no sequenced txs of its own) thus -inherits the prior head rather than recording genesis — otherwise catch-up from -its promoted snapshot would replay the whole stream and double-apply it. - -The same row records `executed_input_count` = storage-derived live head `H`. -The lane passes the count embedded in the just-dumped application; -`insert_pending_dump_in` asserts it equals the maximum current canonical -execution attribution (or era base `K`). This check is inside the -seal/open/snapshot transaction, so a disagreement cannot produce either a -sealed batch or a registered checkpoint. - -## 4. The resume checkpoint - -On startup the lane selects **one** checkpoint (`catch_up_snapshot`, in -`catch_up.rs`): the latest pending snapshot if any, else finalized. The *same* -row supplies `A::from_dump(&prefix)`, physical catch-up cursor -`l2_tx_index`, and canonical `executed_input_count`. Before replay, startup -requires the loaded application's count to equal the stored count. During -replay, each executable physical row must carry exactly the app's current -count, while our batch-envelope rows must carry no mapping. These checks happen -before executing the row and make missing, extra, or wrong attribution a -terminal invariant failure rather than a repair/backfill path. Loading from a -*pending* (not-yet-L1-confirmed) -snapshot is safe because danger-zone recovery clears any cascade-doomed pending -**before** the lane starts (§8) — a surviving pending is either gold or -legitimately in-flight under the optimistic model. - -## 5. Promotion - -When the lane's five-safe-block clock criterion admits an L1-reconciliation -turn (`maybe_advance_safe_frontier`), it walks the complete accumulated -newly-safe range. For each input that is one of *our* batches landing on L1, -`accepted_batch_nonce_at` (reading `safe_accepted_batches`, the -scheduler-acceptance view) yields its nonce. A `BlockObservation` accumulates -the **highest accepted nonce seen in the range and the L1 block it landed in**. -At range close the lane promotes that one `(nonce, block)` target. - -`promote_finalized` points the singleton `finalized_snapshot` at the pending -dump for `max_nonce`, carries over its `l2_tx_index` and -`executed_input_count`, and **deletes every -pending row with `nonce <= max_nonce`** — the promoted one plus any stale rows -behind it. - -### Per-range, not per-block - -Promotion happens **once per eligible clock/reconciliation turn**, even when -the range spans several L1 blocks with several of our batches. Safe-head -observations below the five-block threshold accumulate without draining or -promotion. This is sound, and loses nothing, because of two facts: - -- **Monotonic landing order.** L1 wallet nonces guarantee a higher nonce lands - in a later-or-equal block, so the range's max nonce sits in its *latest* - block-with-our-batch, and `promote_finalized`'s `delete <= max` supersedes - every lower pending. Per-block promotion would compute the *same* `(nonce, - block)` pairs and end at the same final one — it would just expose the - intermediate ones transiently. -- **The intermediate checkpoints were never observable.** `finalized` is a - single row the watchdog polls *asynchronously* — even with per-block - promotion it can miss intermediates between polls. So "visits every block" was - never a guarantee; per-range removes a cadence nicety, not a contract. The - five-block clock intentionally makes multi-block ranges normal, and a delayed - or epoch-sized safe-head jump may make them larger. Finalized state advances - directly to the latest accepted landing in the range; no intermediate - checkpoint is synthesized. - -`BlockObservation` (`snapshot.rs`) keeps one `Option<(nonce, block)>` for -promotion and the direct-execution receipts for the complete reconciliation -range. That vector is required to attach each canonical offset in the eventual -atomic frame transaction. It is confined to the deliberately slow L1 regime; -the user-op hot path does not use it, and scratch paging may bound input reads -without turning the logical reconciliation turn into resumable state. - -### Atomic with the drain - -The promotion is **folded into the same transaction that advances the drain**: -`maybe_advance_safe_frontier` calls -`close_frame_only_with_executions`, which sequences the drained safe -inputs, attaches their canonical execution offsets, rotates the frame, and -runs `promote_finalized_in`—all in one `write`. A crash therefore leaves -promote + delete-pending + drain-sequence + attribution either all committed -or all rolled back. This is the fix for the wedge in §6; see there for why a -*separate* promotion is dangerous. - -The standalone `Storage::promote_finalized` is retained only for test setup -(it's the only way to *supersede* an existing finalized row, which -`insert_finalized_dump` — the genesis-only path — cannot). - -## 6. Case study: the promote/drain wedge - -The earlier design promoted **per block, each in its own transaction**, before a -separate `close_frame_only` advanced the drain. That window was a latent -fail-loud crash-loop. The mechanism, with every escape that *fails* to save it: - -A crash after a promotion commits but before the drain advances leaves a -**promoted-but-undrained** batch. On restart the lane re-processes the same safe -input and re-promotes — on a pending row the first promotion already deleted. -Why nothing prevents that: - -1. **`promote_finalized` hard-fails on a missing pending row** — `SELECT dump_id - … WHERE nonce = ?` → `QueryReturnedNoRows`, no idempotency guard. -2. **`safe_accepted_batches` survives the crash** — the lane *read* it to decide - to promote, so it was committed earlier (by the safe-head sync), independent - of the drain. -3. **`accepted_batch_nonce_at` has no pending-row gate** — bare `SELECT nonce - FROM safe_accepted_batches WHERE safe_input_index = ?`. It still returns the - nonce after the pending row is gone. -4. **The drain cursor didn't move** — `next_undrained_safe_input_index` is - `MAX(safe_input_index)+1` over *sequenced* rows, and sequencing is - `close_frame_only`'s job, which never committed. So the range re-processes. -5. **Recovery doesn't reconcile it** — both recovery paths clear pending *only* - `if !invalidated.is_empty()`, i.e. only on a real cascade. A plain crash - invalidates nothing. - -Result: restart → re-process → `QueryReturnedNoRows` → `InclusionLaneError`, -uncaught → lane exits → next restart hits the identical row → **crash-loop**, no -automatic way out (the promoted batches are L1-confirmed/gold, so even aging -won't trip a cascade to clean them). The window is *wide* during catch-up — it -spans from the first promotion until `close_frame_only` commits. - -**The fix** (§5): fold the single per-range promotion into `close_frame_only`'s -transaction. The "committed promotion, uncommitted drain" state becomes -unrepresentable. - -### Why "missing a promotion" is fine but the wedge is not - -These are opposite crash outcomes, and only one is benign: - -- **Lag** (a promotion that simply didn't happen) is fine: `finalized` stays at - a valid *older* block-complete checkpoint; the lane's own catch-up loads from - the **latest pending** (not finalized), so app state doesn't even lag; and - re-processing re-promotes forward. Converges. -- **Stuck** (re-promoting a deleted row) is the wedge: it makes *no* forward - progress. - -The atomic fold converts every crash into the first kind: either fully -committed, or fully redone cleanly on restart. The invariant it establishes — **a -committed promotion implies an advanced drain past that batch** — is exactly what -the regression tests pin. - -### Regression tests - -- `promotion_advances_drain_atomically_so_restart_cannot_re_promote` — red - against the per-block design (literal `QueryReturnedNoRows`), green after the - fold (promotion + drain advance commit together). -- `close_frame_only_promoting_rolls_back_the_drain_when_promotion_fails` — the - atomicity complement: a mid-tx promotion failure rolls the drain back too. - -## 7. Garbage collection - -A dump becomes collectable when `lease_count = 0` AND it is referenced by -neither `pending_snapshots` nor `finalized_snapshot`. Promotion is what *creates* -such garbage (the superseded finalized, lower-nonce pendings). - -### When GC runs - -**After a promoting clock/reconciliation turn, on the lane's own thread** -(`maybe_advance_safe_frontier`, right after -`close_frame_only_with_executions` -commits — `run_gc::` when a promotion occurred). One full -`gc_unreferenced_dumps` pass per turn that promoted; it reclaims the -just-superseded finalized plus any earlier lease-released garbage. - -Why this, and not the alternatives: - -- **Not the idle path.** GC used to run on the lane's idle branch, gated to 60s - *checked only when idle*. Under sustained load the lane never idles, so GC - starved exactly when batches and their superseded dumps piled up. Tying GC to - *promotion* couples it to garbage *creation*: promotion is ≤ batch-close - frequency, so the cadence self-scales and can't be starved. -- **Not a dedicated worker.** Keeping GC on the lane preserves "every - snapshot-table write happens on one thread." A separate GC worker would be a - second writer contending for the SQLite write lock with the lane's - promote/insert — buying decoupling we don't need at the cost of contention. -- **Not on promotion's "critical path" in any harmful sense.** Promotion is on - the finalized-tracking (safe-frontier) path, not the soft-confirmation hot - path, so a small extra tx there is invisible to user-facing latency. - -`snapshot_gc_at_startup` remains the once-per-boot backstop. If GC ever becomes -expensive, a dedicated background task is still the upgrade path. - -### Crash ordering: no SQLite row pointing at a missing file - -The invariant is **no `dumps` row referencing a non-existent directory**. It -gives the create/delete orderings: - -- **File create → SQLite insert** (file-first): `create_dump` `fsync`s before - the reference row is written. -- **SQLite delete → file delete** (SQLite-first): `gc_unreferenced_dumps` - deletes the rows inside one `write` tx and *returns* the prefixes; the lane's - `run_gc` then removes the enclosing dump directories recursively after - commit. If a directory delete fails, an orphan file is acceptable — the next - startup's `sweep_orphan_dumps` - catches it. The reverse ordering would leave a dangling row. - -This is why `storage/snapshot_dumps.rs` is SQLite-only and the FS half lives in -the lane: the boundary *is* the ordering guarantee. - -### Single-transaction GC closes a lease race - -`gc_unreferenced_dumps` does the eligibility query and the deletes in **one -Immediate-mode tx**. A naive read-then-delete split would race a concurrent -`acquire_*_lease` from an HTTP handler (§ leases below); doing both in one write -serializes against any concurrent writer. - -### Leases (HTTP serving) - -The streaming endpoints (`/finalized_state`, `/latest_snapshot`) must not have -their dump GC'd mid-response. The lease read and the row read are **one atomic -tx** (`acquire_finalized_lease` / `acquire_latest_snapshot_lease`), and the -release guard is armed only after that transaction commits — a failed commit -cannot schedule a decrement for an increment that rolled back. The handler then -holds the lease for the response lifetime via the **drop-guard** inside the -streaming body, so it releases on completion, error, *and* client disconnect. -Releases are enqueued to a **supervised** blocking task set -(`http.rs::supervise_snapshot_releases`) that the HTTP worker drains before -ordinary shutdown completes. Terminal failures abort without draining. A -release failure is classified like any storage failure: a *persistent* error -(e.g. the lease row is gone — `StatementChangedRows` — or a persistent -open/migration failure) is a storage-invariant violation and takes the -runtime down terminally (SIGABRT); transient failures (BUSY, I/O) are logged -and left to the startup backstop. `reset_dump_leases` at startup remains the -crash backstop for releases that never ran. (Endpoint shapes: -[`AGENTS.md`](../../AGENTS.md) and the root [`README.md`](../../README.md).) - -### Startup sequence - -Before this sequence, startup recovery has already required a finalized -snapshot fact and established a Tip through either guarded `ensure_open_tip_for_recovery` or -an atomic recovery reopen. `PreparedRuntime::prepare` then calls -`startup_hygiene::run_snapshot_hygiene`, which runs five order-critical -steps before runtime admission, while no task -exists: (1) `reset_dump_leases` (clear stale leases from a crashed run), -(2) `require_finalized_snapshot`, (3) `restamp_finalized_promotion`, -(4) `snapshot_gc_at_startup`, and (5) `sweep_orphan_dumps` (remove on-disk dirs -not in `dumps`; the finalized prefix is already registered and cannot be -swept). Final admission and the non-yielding worker launch follow only after -preparation completes and admission re-inspects current facts. - -## 8. Recovery interaction - -Danger-zone recovery (`storage/recovery.rs`, see -[`../recovery/README.md`](../recovery/README.md)) cascade-invalidates batches -that the canonical stream will never reach. In the same transaction as the -cascade it clears `pending_snapshots` **scoped to the cascade**: only rows -with `nonce >= pivot.nonce` — exactly the cascaded batches' pendings, which -catch-up must never load (`cascade_and_reopen`, the shared tail of both -recovery paths). - -The same cascade retains the physical `sequenced_l2_txs` audit rows but deletes -their derived `executed_inputs` mappings, advances `RecoveryGeneration` once, -and opens the replacement Tip atomically. The surviving snapshot count is the -retained logical head; replacement history reuses the rewound suffix offsets -under the new generation. A crash cannot expose a new generation with old -mappings, or a rewound projection with doomed pending state. - -Pendings of *gold but not-yet-promoted* batches (landed and accepted while -the process was down) carry lower nonces and **survive**: catch-up resumes -from the freshest surviving checkpoint, and the rows are cleaned up by the -next promotion's `DELETE <= max_nonce`. The scoping makes the §6 -promote-wedge **unrepresentable** rather than unreachable: any nonce the lane -can later observe as accepted either has its pending row intact or belongs -to a post-recovery batch with a fresh row. (The earlier blanket clear was -safe only through a chain of cross-file couplings — same-tx full-backlog -reopen drain, `check_danger` arm ordering, frame-safe-block -monotonicity.) In the `RecoverTip` path the -scope deletes nothing: the Tip never has a pending row. - -`finalized` is untouched (its bytes are for an L1-confirmed batch, which -survives any cascade). A **no-op** recovery (closed batches gold, Tip fresh) -deliberately preserves in-flight pendings the lane is still working with. -This is why catch-up can safely resume from a surviving pending (§4). - -## 9. Where the code lives - -| Concern | Location | -|---------------------------------|----------| -| Dump trait + wire format | [`format.md`](format.md); `sequencer-core/src/application/`, `examples/app-core/` | -| Storage (SQLite only) | `sequencer/src/storage/snapshot_dumps.rs`; atomic close + promote in `storage/ingress.rs` | -| Lane integration (take/observe/GC) | `sequencer/src/ingress/inclusion_lane/snapshot.rs`, `mod.rs`, `catch_up.rs` | -| Runtime startup sequence | `sequencer/src/commands/run/startup_hygiene.rs` (called from `commands/run/workers.rs`) | -| HTTP serving + leases | `sequencer/src/egress/api/snapshot.rs` | -| Recovery clear | `sequencer/src/storage/recovery.rs` | - -## 10. Deferred / future work - -- **Watchdog (separate project).** `/finalized_state` and - `/finalized_state/inclusion_block` are consumed by an operator watchdog that - advances its own canonical machine through L1 to the served `inclusion_block` - and compares its `inspect_state` output to the served bytes. Its prerequisite - is a real `inspect_state` on the canonical-machine app — the symmetric side of - `create_dump` (see [`format.md`](format.md)) — currently a stub in - `examples/canonical-app/`. The watchdog itself lives outside this repo. -- **Pending-pool upper bound.** The pending pool grows per batch-close until - promotion; pathologically (L1 stops accepting batches) it grows unboundedly. - Harmless for the toy wallet's tiny dumps; cap + reject + alert once a real - app's state is hundreds of MB. -- **Directory-style dumps.** `create_dump` already supports multi-file prefixes, - but the HTTP layer streams a single `state_file_in_dump`. Serving a directory - means a tar-stream or a minimal archive format. (Lease design "X" already - holds for the whole stream, so no rework there.) -- **`Range:` requests / compression** for `/finalized_state` — defer until - measured; state bytes are low-entropy but range-resume of large dumps may - matter. -- **Cross-implementation test vectors** — land when a second `Application` (the - canonical machine's) exists to validate byte-for-byte against the wallet's - format. +The application owns `state`, the prefix passed to `Application::create_dump` +and `Application::from_dump`, and must support independent restoration after +source deletion. Its canonical comparison file may be only one part of that +artifact. The [Application contract](../protocol/application-contract.md#6-checkpoint-lifecycle) +owns durability and engine behavior; [format.md](format.md) describes the wallet. + +SQLite separates artifact ownership from snapshot boundaries: + +| Fact | Meaning | +|---|---| +| `dumps` | Artifact path and active reader lease count | +| `snapshots` | Artifact's immutable application count and local batch identity | +| Baseline snapshot (`batch_index IS NULL`) | State from which this era starts | +| `safe_accepted_batches` | Scheduler-accepted, content-matched local batch landings | + +The application count is the next canonical input position, never a physical +SQLite cursor. The batch identity disambiguates empty batches, which can share +an application count, and replacement branches, which can reuse a nonce. + +## Creation and restart + +Initially every batch close creates a snapshot. The lane makes the artifact +and metadata durable **before** one transaction seals the batch and registers +its snapshot. A failed database commit leaves an orphan directory; startup +sweeps it. A committed close therefore has its required snapshot. + +Setup similarly makes the baseline artifact durable before publishing the +complete era baseline, recovery root when applicable, and setup-completion +facts atomically. + +Restart selects the newest snapshot on the valid batch branch, or the baseline +when no batch snapshot exists. The selected artifact and stored application +count come from one row. Catch-up checks the restored engine's count and replays +application inputs from that count. Invalidated branches are excluded by the +same valid-batch relation used elsewhere. + +## Acceptance and comparison + +The newest accepted batch determines the comparison checkpoint. Its snapshot +must exist; storage refuses a missing required row instead of falling back to an +older snapshot. Acceptance already includes scheduler validation and local +content identity, so merely observing an own-sender L1 input is insufficient. + +This selection is independent of the lane's L1 reconciliation cursor. A crash +between reader ingestion and lane reconciliation cannot miss a promotion or +repeat one: acceptance is already durable, and the query derives the result. + +The genesis baseline is known canonical at block zero. A rebuilt baseline is +**only a restore artifact** until a new batch is accepted. The recovery fold can +pre-execute queued direct inputs through its stop block; that state need not +equal the canonical machine's state at that block. It is never advertised as an +accepted comparison checkpoint solely because recovery produced it. + +The current watchdog compares at the selected accepted batch's **L1 block +boundary**. Every batch has a snapshot, and reader ingestion accounts for a +complete safe block before publishing its accepted prefix. Selecting the latest +accepted batch therefore includes later accepted batches in the same block. +Sparse snapshots are a future policy change: an older artifact cannot be +labelled as that block's final state when a later accepted batch in the block +has no artifact. That change must settle comparison positioning and replay +retention together. + +## HTTP and recovery exports + +These endpoints are operator-only and require network isolation: + +- `/latest_snapshot` streams a tar archive containing `info.toml` and the complete + opaque `state` artifact. It may describe optimistic state. +- `/finalized_state` streams only the canonical comparison file for the latest + accepted checkpoint. `/finalized_state/inclusion_block` provides its block and + executed-input count for the watchdog. +- `/finalized_snapshot` streams a complete recovery tar archive containing + `info.toml`, `state`, and a generated `checkpoint.toml` acceptance receipt. + The receipt supplies the accepted inclusion block and next batch nonce. + +Snapshot bodies carry `X-History-Era`, `X-Recovery-Generation`, and +`X-Executed-Input-Count`. Acceptance endpoints also carry `X-Inclusion-Block`. +The metadata, selected artifact, and lease are captured in one transaction. +Restoring `/latest_snapshot` and subscribing with its history claim gives the +consumer a coherent snapshot-plus-suffix starting point. + +**Operator backup workflow:** download `/finalized_snapshot` and extract the +archive. Supply that extracted directory to `setup --recovery`, with the +checkpoint block recorded in its receipt. The loader checks the receipt against +the immutable dump metadata and configured block. Copying a bare local +`dumps//` directory is insufficient: it is a restore artifact and has no +acceptance receipt. Export creates the receipt without modifying local files; +startup no longer stamps or repairs acceptance metadata in `info.toml`. + +## Retention, leases, and crash safety + +Garbage collection retains: + +1. The newest accepted checkpoint, or the baseline before first acceptance. +2. Every valid snapshot beyond the accepted frontier. An intermediate optimistic + batch may become the next accepted head before its successors do. +3. Every artifact with an active reader lease. + +Older accepted artifacts and invalidated branch artifacts are collectible. +The baseline artifact can be retired after an accepted checkpoint replaces its +rollback role; the era's immutable baseline metadata remains in SQLite. + +Selection, lease acquisition, and GC serialize through SQLite write +transactions. A lease release guard is armed only after its increment commits. +An HTTP body retains the lease through completion or disconnect. Archive +production also retains ownership until it stops reading the source, so dropping +the network stream cannot race producer reads against filesystem deletion. + +GC selects and deletes eligible database rows in one transaction, then removes +the enclosing dump directories recursively. This filesystem operation disposes +of all checkpoint resources. Filesystem deletion failure leaves a harmless +orphan for the startup sweep. The reverse ordering would leave a durable row pointing at missing +state and is forbidden. Startup clears leases left by the dead process, +validates a rollback checkpoint, collects obsolete rows, and sweeps orphan +directories before workers start. Missing or corrupt referenced artifacts fail +loud; operational filesystem errors retain their normal error classification. diff --git a/docs/watchdog/README.md b/docs/watchdog/README.md index c1181fed..23f90364 100644 --- a/docs/watchdog/README.md +++ b/docs/watchdog/README.md @@ -139,8 +139,8 @@ the Lua side without also porting the version witness that makes it sound. The sequencer exposes operator-internal snapshot routes (see `sequencer/src/egress/api/snapshot.rs`): -- `GET /finalized_state/inclusion_block` — cheap JSON `{ inclusion_block, l2_tx_index }` polled every compare tick. -- `GET /finalized_state` — streams the finalized SSZ state file (`application/octet-stream`) with `X-Inclusion-Block` and `X-L2-Tx-Index` headers. +- `GET /finalized_state/inclusion_block` — cheap JSON `{ inclusion_block, executed_input_count }` polled every compare tick. +- `GET /finalized_state` — streams the finalized SSZ state file (`application/octet-stream`) with `X-Inclusion-Block` and `X-Executed-Input-Count` headers. **Idle optimization:** when `inclusion_block` has not advanced past the watchdog checkpoint's `safe_block`, the tick returns diff --git a/docs/watchdog/getting-started.md b/docs/watchdog/getting-started.md index 62311b5f..17d85058 100644 --- a/docs/watchdog/getting-started.md +++ b/docs/watchdog/getting-started.md @@ -119,7 +119,7 @@ In another shell (use the printed `CARTESI_WATCHDOG_SEQUENCER_URL`): curl -s "$CARTESI_WATCHDOG_SEQUENCER_URL/finalized_state/inclusion_block" ``` -When you see JSON like `{"inclusion_block":0,"l2_tx_index":0}` (numbers may differ), the watchdog can compare. If it stays 404 for a long time, check sequencer logs in `tests/e2e/results/` and that L1 is mining (devnet Anvil auto-mines by default). +When you see JSON like `{"inclusion_block":0,"executed_input_count":0}` (numbers may differ), the watchdog can compare. If it stays 404 for a long time, check sequencer logs in `tests/e2e/results/` and that L1 is mining (devnet Anvil auto-mines by default). Optional — inspect SSZ size: diff --git a/sdk/rust-client/Cargo.toml b/sdk/rust-client/Cargo.toml index 32b5a5a3..10cf1958 100644 --- a/sdk/rust-client/Cargo.toml +++ b/sdk/rust-client/Cargo.toml @@ -16,3 +16,6 @@ serde_json = { workspace = true } thiserror = { workspace = true } tokio = { workspace = true } tokio-tungstenite = { workspace = true } + +[dev-dependencies] +tokio = { workspace = true, features = ["macros", "rt", "io-util", "net", "time"] } diff --git a/sdk/rust-client/src/errors.rs b/sdk/rust-client/src/errors.rs index 69c3ff08..ec35b321 100644 --- a/sdk/rust-client/src/errors.rs +++ b/sdk/rust-client/src/errors.rs @@ -69,8 +69,18 @@ pub enum GetFeeError { #[derive(Debug, Error)] pub enum SubscribeError { + #[error(transparent)] + History(#[from] sequencer_core::history::HistoryPolicyError), #[error("invalid endpoint: {0}")] InvalidEndpoint(String), #[error("ws connect failed: {0}")] Connect(String), } + +#[derive(Debug, Error)] +pub enum SnapshotError { + #[error("snapshot request failed: {0}")] + Request(#[from] reqwest::Error), + #[error("invalid snapshot metadata: {0}")] + Metadata(String), +} diff --git a/sdk/rust-client/src/lib.rs b/sdk/rust-client/src/lib.rs index 734a884e..7003c5b0 100644 --- a/sdk/rust-client/src/lib.rs +++ b/sdk/rust-client/src/lib.rs @@ -3,7 +3,13 @@ mod errors; -pub use errors::{ClientBuildError, GetFeeError, SubmitRejected, SubmitTxError, SubscribeError}; +pub use errors::{ + ClientBuildError, GetFeeError, SnapshotError, SubmitRejected, SubmitTxError, SubscribeError, +}; + +pub use sequencer_core::history::{ + ExecutedInputCount, HistoryClaim, HistoryPolicyError, HistoryVersion, +}; use sequencer_core::api::{FeeResponse, TxRequest, TxResponse}; use std::time::Duration; @@ -12,6 +18,13 @@ use tokio_tungstenite::{MaybeTlsStream, WebSocketStream, connect_async}; pub type SubscribeStream = WebSocketStream>; +/// Metadata and streaming body from the same leased checkpoint response. +/// Restore the archive before using `claim` to resume its application history. +pub struct SnapshotResponse { + pub claim: HistoryClaim, + pub response: reqwest::Response, +} + #[derive(Debug, Clone)] pub struct SequencerClient { endpoint: String, @@ -64,10 +77,10 @@ impl SequencerClient { self } - pub fn ws_subscribe_url(&self, from_offset: u64) -> String { - with_from_offset( + pub fn ws_subscribe_url(&self, claim: HistoryClaim) -> String { + with_history_claim( default_ws_subscribe_url_for_http(self.endpoint.as_str()).as_str(), - from_offset, + claim, ) } @@ -80,6 +93,7 @@ impl SequencerClient { let response = self .http_client .post(&url) + .timeout(self.request_timeout) .json(req) .send() .await @@ -107,6 +121,7 @@ impl SequencerClient { let response = self .http_client .get(&url) + .timeout(self.request_timeout) .send() .await .map_err(map_reqwest_error)?; @@ -121,11 +136,55 @@ impl SequencerClient { serde_json::from_str::(&body).map_err(|e| GetFeeError::Decode(e.to_string())) } - pub async fn subscribe(&self, from_offset: u64) -> Result { - let url = self.ws_subscribe_url(from_offset); + /// Streams without the short transaction deadline; callers own download cancellation. + pub async fn latest_snapshot(&self) -> Result { + let response = self + .http_client + .get(format!( + "{}/latest_snapshot", + self.endpoint.trim_end_matches('/') + )) + .send() + .await? + .error_for_status()?; + let header = |name| { + response + .headers() + .get(name) + .ok_or_else(|| SnapshotError::Metadata(format!("missing {name}")))? + .to_str() + .map_err(|error| SnapshotError::Metadata(error.to_string())) + }; + let era_id = header("X-History-Era")?.parse().map_err( + |error: sequencer_core::history::EraIdParseError| { + SnapshotError::Metadata(error.to_string()) + }, + )?; + let generation: u64 = header("X-Recovery-Generation")? + .parse() + .map_err(|error: std::num::ParseIntError| SnapshotError::Metadata(error.to_string()))?; + let count: u64 = header("X-Executed-Input-Count")? + .parse() + .map_err(|error: std::num::ParseIntError| SnapshotError::Metadata(error.to_string()))?; + Ok(SnapshotResponse { + claim: HistoryClaim { + version: HistoryVersion { + era_id, + recovery_generation: sequencer_core::history::RecoveryGeneration::new( + generation, + ), + }, + next_input: ExecutedInputCount::new(count), + }, + response, + }) + } + + pub async fn subscribe(&self, claim: HistoryClaim) -> Result { + let url = self.ws_subscribe_url(claim); let (stream, _response) = connect_async(url.as_str()) .await - .map_err(|e| SubscribeError::Connect(e.to_string()))?; + .map_err(map_subscribe_error)?; Ok(stream) } } @@ -133,7 +192,6 @@ impl SequencerClient { fn build_http_client(request_timeout: Duration) -> Result { reqwest::Client::builder() .connect_timeout(request_timeout) - .timeout(request_timeout) .pool_max_idle_per_host(64) .build() } @@ -175,11 +233,128 @@ fn default_ws_subscribe_url_for_http(http_url: &str) -> String { format!("{}/ws/subscribe", scheme_replaced.trim_end_matches('/')) } -fn with_from_offset(ws_subscribe_url: &str, from_offset: u64) -> String { +fn with_history_claim(ws_subscribe_url: &str, claim: HistoryClaim) -> String { let separator = if ws_subscribe_url.contains('?') { '&' } else { '?' }; - format!("{ws_subscribe_url}{separator}from_offset={from_offset}") + format!( + "{ws_subscribe_url}{separator}era_id={}&recovery_generation={}&next_input={}", + claim.version.era_id, + claim.version.recovery_generation.get(), + claim.next_input.get() + ) +} + +fn map_subscribe_error(error: tokio_tungstenite::tungstenite::Error) -> SubscribeError { + if let tokio_tungstenite::tungstenite::Error::Http(response) = &error + && response.status().as_u16() == 409 + && let Some(bytes) = response + .headers() + .get("X-History-Error") + .map(|value| value.as_bytes()) + .or_else(|| response.body().as_deref()) + && let Ok(policy) = serde_json::from_slice::(bytes) + { + return SubscribeError::History(policy); + } + SubscribeError::Connect(error.to_string()) +} + +#[cfg(test)] +mod tests { + use super::*; + use sequencer_core::history::{EraId, RecoveryGeneration}; + + #[tokio::test] + async fn fee_request_keeps_its_request_timeout() { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let client = SequencerClient::new_with_timeout( + format!("http://{address}"), + Duration::from_millis(100), + ) + .unwrap(); + let request = client.get_fee(); + let stalled_server = async { + let (_stream, _) = listener.accept().await.unwrap(); + std::future::pending::<()>().await; + }; + let result = tokio::time::timeout(Duration::from_secs(5), async { + tokio::select! { + result = request => result, + () = stalled_server => unreachable!(), + } + }) + .await + .expect("fee request must retain its deadline independently of snapshot streaming"); + assert!(matches!( + result, + Err(GetFeeError::Transport(SubmitTxError::TimeoutRead)) + )); + } + + #[tokio::test] + async fn snapshot_body_outlives_the_transaction_request_timeout() { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { + let (mut stream, _) = listener.accept().await.unwrap(); + let mut request = [0; 4096]; + let mut received = 0; + while !request[..received].ends_with(b"\r\n\r\n") { + let count = stream.read(&mut request[received..]).await.unwrap(); + assert_ne!(count, 0, "complete request headers"); + received += count; + } + stream.write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 4\r\nX-History-Era: 00112233-4455-4677-8899-aabbccddeeff\r\nX-Recovery-Generation: 0\r\nX-Executed-Input-Count: 7\r\n\r\n").await.unwrap(); + tokio::time::sleep(Duration::from_millis(100)).await; + stream.write_all(b"dump").await.unwrap(); + }); + let client = SequencerClient::new_with_timeout( + format!("http://{address}"), + Duration::from_millis(30), + ) + .unwrap(); + let snapshot = client.latest_snapshot().await.unwrap(); + assert_eq!(snapshot.claim.next_input.get(), 7); + assert_eq!(snapshot.response.bytes().await.unwrap().as_ref(), b"dump"); + server.await.unwrap(); + } + + #[test] + fn subscription_url_carries_the_exact_resume_claim() { + let claim = HistoryClaim { + version: HistoryVersion { + era_id: "00112233-4455-4677-8899-aabbccddeeff" + .parse::() + .unwrap(), + recovery_generation: RecoveryGeneration::new(9), + }, + next_input: ExecutedInputCount::new(50_001), + }; + let client = SequencerClient::new("http://localhost:8080").unwrap(); + assert_eq!( + client.ws_subscribe_url(claim), + "ws://localhost:8080/ws/subscribe?era_id=00112233-4455-4677-8899-aabbccddeeff&recovery_generation=9&next_input=50001" + ); + } + + #[test] + fn typed_refusal_survives_an_http_body_in_a_later_packet() { + let policy = HistoryPolicyError::AheadOfHead { + head: ExecutedInputCount::new(12), + }; + let response = tokio_tungstenite::tungstenite::http::Response::builder() + .status(409) + .header("X-History-Error", serde_json::to_string(&policy).unwrap()) + .body(Some(Vec::new())) + .unwrap(); + assert!( + matches!(map_subscribe_error(tokio_tungstenite::tungstenite::Error::Http(Box::new(response))), + SubscribeError::History(actual) if actual == policy) + ); + } } diff --git a/sequencer-core/src/history.rs b/sequencer-core/src/history.rs index 3125aa21..dbd1b7fa 100644 --- a/sequencer-core/src/history.rs +++ b/sequencer-core/src/history.rs @@ -4,7 +4,7 @@ //! External history identity and version coordinates. use serde::{Deserialize, Serialize}; -use std::fmt; +use std::{fmt, str::FromStr}; use thiserror::Error; /// Boundary before the next canonical application input executes. @@ -44,10 +44,8 @@ impl ExecutedInputCount { /// One durable setup/rebuild era. /// -/// The bytes must carry the RFC 4122 UUIDv4 version and variant bits. Display -/// uses the canonical lowercase hyphenated representation. The wire (text / -/// JSON) codec deliberately does not exist yet: Track 3 owns the wire -/// projection and adds it beside its consumer when that lands. +/// The bytes must carry the RFC 4122 UUIDv4 version and variant bits. Text and +/// JSON use the canonical lowercase hyphenated representation. #[derive(Clone, Copy, PartialEq, Eq, Hash)] pub struct EraId([u8; 16]); @@ -95,6 +93,33 @@ impl fmt::Display for EraId { } } +impl FromStr for EraId { + type Err = EraIdParseError; + + fn from_str(value: &str) -> Result { + if value.len() != 36 || [8, 13, 18, 23].iter().any(|&i| value.as_bytes()[i] != b'-') { + return Err(EraIdParseError::InvalidText); + } + let hex = value.replace('-', ""); + let bytes = alloy_primitives::hex::decode(hex).map_err(|_| EraIdParseError::InvalidText)?; + Self::try_from(bytes.as_slice()) + } +} + +impl Serialize for EraId { + fn serialize(&self, serializer: S) -> Result { + serializer.collect_str(self) + } +} + +impl<'de> Deserialize<'de> for EraId { + fn deserialize>(deserializer: D) -> Result { + String::deserialize(deserializer)? + .parse() + .map_err(serde::de::Error::custom) + } +} + impl fmt::Debug for EraId { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_tuple("EraId").field(&self.to_string()).finish() @@ -103,6 +128,8 @@ impl fmt::Debug for EraId { #[derive(Debug, Clone, Copy, PartialEq, Eq, Error)] pub enum EraIdParseError { + #[error("era id must be a hyphenated UUIDv4")] + InvalidText, #[error("era id blob has length {actual}, expected 16")] InvalidByteLength { actual: usize }, #[error("era id is not UUID version 4")] @@ -129,22 +156,22 @@ impl RecoveryGeneration { } /// Equality/discontinuity token for locally available application history. -/// Like [`EraId`], its wire form is Track 3's to define beside its consumer. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +/// Consumers must claim both fields when resuming application history. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] pub struct HistoryVersion { pub era_id: EraId, pub recovery_generation: RecoveryGeneration, } /// The history a consumer holds and the next application input it can execute. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] pub struct HistoryClaim { pub version: HistoryVersion, pub next_input: ExecutedInputCount, } /// One coherent view of the locally available canonical history. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] pub struct HistoryBounds { pub version: HistoryVersion, pub available_from: ExecutedInputCount, @@ -182,7 +209,8 @@ impl HistoryBounds { } } -#[derive(Debug, Clone, Copy, PartialEq, Eq, Error)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Error, Serialize, Deserialize)] +#[serde(tag = "code", rename_all = "SCREAMING_SNAKE_CASE")] pub enum HistoryPolicyError { #[error("history era changed")] EraChanged { current: HistoryVersion }, @@ -204,6 +232,32 @@ mod tests { 0x00, ]; + #[test] + fn era_and_claim_json_preserve_their_exact_identity() { + let era: EraId = "00112233-4455-4677-8899-aabbccddeeff".parse().unwrap(); + assert_eq!( + serde_json::to_string(&era).unwrap(), + "\"00112233-4455-4677-8899-aabbccddeeff\"" + ); + let claim = HistoryClaim { + version: HistoryVersion { + era_id: era, + recovery_generation: RecoveryGeneration::new(7), + }, + next_input: ExecutedInputCount::new(u64::MAX), + }; + assert_eq!( + serde_json::from_str::(&serde_json::to_string(&claim).unwrap()).unwrap(), + claim + ); + assert!( + "00112233-4455-1677-8899-aabbccddeeff" + .parse::() + .is_err() + ); + assert!("00112233445546778899aabbccddeeff".parse::().is_err()); + } + #[test] fn era_id_displays_canonical_lowercase_hyphenated_form() { let era = EraId::from_bytes(CANONICAL_BYTES).expect("canonical UUIDv4"); diff --git a/sequencer/Cargo.toml b/sequencer/Cargo.toml index 408a1ba0..63651867 100644 --- a/sequencer/Cargo.toml +++ b/sequencer/Cargo.toml @@ -13,10 +13,11 @@ authors.workspace = true sequencer-core = { path = "../sequencer-core" } axum = { version = "0.8.8", features = ["ws"] } tokio = { workspace = true, features = ["macros", "rt-multi-thread", "sync", "time", "net", "signal", "fs", "io-util"] } -tokio-util = { version = "0.7", features = ["io"] } +tokio-util = { version = "0.7", features = ["io", "io-util"] } serde = { workspace = true } serde_json = { workspace = true } toml = "0.8" +tar = "0.4" tracing = { workspace = true } tower-http = { version = "0.6.8", features = ["trace", "cors"] } rusqlite = { workspace = true } diff --git a/sequencer/src/commands/config.rs b/sequencer/src/commands/config.rs index 6aab2eeb..3702bdf7 100644 --- a/sequencer/src/commands/config.rs +++ b/sequencer/src/commands/config.rs @@ -377,7 +377,7 @@ pub struct SetupConfig { pub checkpoint_block: u64, /// Recovery mode (cockroach recovery): rebuild this freshly-wiped DB from a /// trusted checkpoint at `--checkpoint-block` instead of genesis bootstrap. - /// Requires `--checkpoint-dump-dir`, `--checkpoint-block > 0`, and the + /// Requires `--checkpoint-dump-dir` and the /// batch-submitter signing key (recovery flushes the wallet nonce, which /// signs L1 no-ops). Plain `setup` stays L1-read-only and key-less. #[arg(long, env = "CARTESI_SEQUENCER_RECOVERY", default_value_t = false)] @@ -410,7 +410,7 @@ impl SetupConfig { /// conditionally-required arg group). Returns the first violation as a /// human-readable message; the caller maps it to a terminal bootstrap error. /// - /// Recovery requires a real checkpoint (`--checkpoint-block > 0` + a dump + /// Recovery requires an exported checkpoint (a matching block + dump /// dir) and the signing key; a plain `setup` must carry none of the /// recovery-only inputs (key, dump dir). pub fn validate(&self) -> Result<(), String> { @@ -426,11 +426,6 @@ impl SetupConfig { if self.checkpoint_dump_dir.is_none() { return Err("--recovery requires --checkpoint-dump-dir".to_string()); } - if self.checkpoint_block == 0 { - return Err("--recovery requires --checkpoint-block > 0 \ - (recovery boots a non-genesis checkpoint)" - .to_string()); - } if !key_present { return Err("--recovery requires the batch-submitter signing key \ (--batch-submitter-private-key[-file]) to flush the wallet nonce" @@ -722,7 +717,7 @@ mod tests { } #[test] - fn recovery_requires_dump_dir_block_and_key() { + fn recovery_requires_dump_dir_and_key() { // --recovery alone (no dump dir / block / key) is invalid; each missing // input is reported. let bare = setup_config_from(&["--recovery"]); @@ -732,13 +727,17 @@ mod tests { .contains("--checkpoint-dump-dir") ); - let no_block = setup_config_from(&["--recovery", "--checkpoint-dump-dir", "/tmp/ckpt"]); - assert!( - no_block - .validate() - .unwrap_err() - .contains("--checkpoint-block > 0") - ); + let genesis = setup_config_from(&[ + "--recovery", + "--checkpoint-dump-dir", + "/tmp/ckpt", + "--batch-submitter-private-key", + TEST_KEY, + ]); + genesis + .validate() + .expect("genesis checkpoint is a valid recovery source"); + assert_eq!(genesis.checkpoint_block, 0); let no_key = setup_config_from(&[ "--recovery", diff --git a/sequencer/src/commands/error.rs b/sequencer/src/commands/error.rs index 0a2034c0..37aaf820 100644 --- a/sequencer/src/commands/error.rs +++ b/sequencer/src/commands/error.rs @@ -407,16 +407,15 @@ pub enum SetupRecoveryError { AlreadySetUp, /// The checkpoint dump could not be loaded (missing/corrupt `info.toml`, or /// the app's `from_dump` failed). Operator must supply a valid **sequencer** - /// dump dir (`info.toml` + `state/`), not a watchdog CM checkpoint. + /// recovery export (`info.toml` + `checkpoint.toml` + `state/`), not a watchdog CM checkpoint. #[error("failed to load checkpoint dump at {path}: {message}")] CheckpointLoad { path: String, message: String }, - /// The checkpoint's last-executed safe block `A` is not strictly before the - /// checkpoint block `B`. The fold reconstructs the `(A, B]` fridge, so - /// `A < B` must hold — otherwise the checkpoint dump and - /// `--checkpoint-block` describe inconsistent points. + /// Outside the known empty genesis checkpoint, A must precede B so the + /// recovery seed includes all potentially pending directs in block B. #[error( - "checkpoint last-executed safe block {executed_safe_block} (A) is not \ - before checkpoint block {checkpoint_block} (B)" + "checkpoint last-executed safe block {executed_safe_block} (A) must precede \ + checkpoint block {checkpoint_block} (B), except for empty genesis; \ + equality can omit pending same-block directs" )] CheckpointNotBeforeBlock { executed_safe_block: u64, @@ -429,58 +428,6 @@ pub enum SetupRecoveryError { internal storage invariant violation" )] MissingResyncedSafeHead, - /// A re-run of `setup --recovery` found a root tip from a *prior* (crashed - /// before setup completion) attempt whose nonce differs from this - /// attempt's resume nonce — a different checkpoint, or the same one after the - /// post-flush head `C` advanced. The half-recovered DB cannot be resumed - /// onto a tree rooted at the old nonce (the anchor would move but the - /// existing root tip would not, silently breaking I16). Wipe the data dir - /// and re-run. - #[error( - "partial recovery: existing root tip carries nonce {existing_root_nonce}, \ - but this attempt resumes at {requested_nonce} — wipe the data dir and re-run" - )] - PartialRecoveryMismatch { - existing_root_nonce: u64, - requested_nonce: u64, - }, - /// A re-run of `setup --recovery` found a root tip carrying *this* attempt's - /// resume nonce but **no finalized snapshot** — a prior attempt that crashed - /// between opening the root tip and writing the snapshot. It cannot be - /// resumed safely: a re-sync may have advanced `C` with new direct inputs - /// (which leave `N'` unchanged) that resuming would leave unsequenced, so the - /// snapshot cursor would lag the folded `S'` and `run` would drain+execute - /// them a second time (divergence). Wipe the data dir and re-run (the - /// one-shot recovery model). - #[error( - "partial recovery: root tip at nonce {root_nonce} exists with no finalized \ - snapshot (crashed mid-fill) — wipe the data dir and re-run" - )] - PartialRecoveryIncomplete { root_nonce: u64 }, - /// `setup --recovery` found a finalized snapshot but **no root tip**. A - /// completed cockroach fill always has both (the tip is opened in step 2, - /// before the snapshot in step 4), so this is residue from a *different* - /// deployment mode left in the data dir — a plain `setup` that registered the - /// genesis finalized snapshot and crashed before setup completion. - /// Folding `(S', N')` and then silently keeping the old snapshot would mark - /// setup complete over the genesis state instead of the recovered state. Wipe - /// the data dir and re-run `setup --recovery`. - #[error( - "setup --recovery found a finalized snapshot (block {existing_finalized_block}) \ - with no root tip — residue from an incomplete plain `setup`; wipe the data \ - dir and re-run" - )] - RecoveryOverResidualSnapshot { existing_finalized_block: u64 }, - /// A plain (non-recovery) `setup` found a non-zero batch-tree anchor — - /// residue from a `setup --recovery` that crashed before completion. Booting - /// a genesis deployment over it would root the tree at the recovery nonce - /// instead of 0. Wipe the data dir, then run plain `setup` or re-run - /// `setup --recovery`. - #[error( - "plain setup found batch-tree anchor {anchor} (≠ 0) — leftover from an \ - incomplete `setup --recovery`; wipe the data dir and re-run" - )] - GenesisOverRecoveryResidue { anchor: u64 }, } /// `setup`'s read-only detection gate: the reasons a @@ -1095,28 +1042,6 @@ mod tests { CommandError::from(SetupRecoveryError::AlreadySetUp), "setup --recovery over an already set-up directory", ), - // Partial-recovery residue: operator must wipe — terminal. - ( - CommandError::from(SetupRecoveryError::PartialRecoveryMismatch { - existing_root_nonce: 3, - requested_nonce: 5, - }), - "partial-recovery residue at a different nonce", - ), - ( - CommandError::from(SetupRecoveryError::GenesisOverRecoveryResidue { anchor: 7 }), - "genesis over recovery residue", - ), - ( - CommandError::from(SetupRecoveryError::PartialRecoveryIncomplete { root_nonce: 3 }), - "an incomplete partial recovery", - ), - ( - CommandError::from(SetupRecoveryError::RecoveryOverResidualSnapshot { - existing_finalized_block: 0, - }), - "recovery over a residual snapshot", - ), // A checkpoint predating genesis is operator misconfig — terminal // (30), not a recovery trigger (40). ( diff --git a/sequencer/src/commands/run/startup_hygiene.rs b/sequencer/src/commands/run/startup_hygiene.rs index 93c47f69..409acaaf 100644 --- a/sequencer/src/commands/run/startup_hygiene.rs +++ b/sequencer/src/commands/run/startup_hygiene.rs @@ -1,46 +1,19 @@ // (c) Cartesi and individual authors (see AUTHORS) // SPDX-License-Identifier: Apache-2.0 (see LICENSE) -//! Startup snapshot hygiene: the authority-neutral repair pass `prepare` -//! runs over the data directory before the runtime boundary. Synchronous, -//! spawns nothing, awaits nothing, holds no `RuntimeScope` — it runs inside -//! `prepare`, before admission, while zero workers exist. -//! -//! [`run_snapshot_hygiene`] runs five steps, in this order: -//! -//! 1. Reset stale leases. A crashed previous run may have left -//! `lease_count > 0` on dumps that aren't being read by anyone now; -//! without this, GC would skip them forever. -//! 2. Require the finalized snapshot (always-load invariant). `setup` -//! registered the genesis snapshot and `run` gated on atomic setup -//! completion, so it must be present — a missing one is a terminal -//! incomplete-setup, not a cold-start to paper over (run holds no -//! genesis app instance). -//! 3. Re-stamp the finalized dump's `info.toml` from the authoritative DB -//! row. Idempotent, and independent of the two cleanup steps below (the -//! finalized dump is referenced, so neither GC nor the sweep can touch -//! it) — what matters is that it lands before the lane loads the dump, -//! because `info.toml` is the sole authority for `setup --recovery`. -//! Missing or corrupt metadata under a DB-referenced row is terminal, -//! never healed. -//! 4. GC SQLite-side: drop any rows now unreferenced after promotions or -//! invalidations that finalized just before the previous shutdown. -//! 5. Orphan FS sweep: remove directories under `dumps_dir` that aren't -//! tracked by SQLite (crash-during-create_dump or -//! crash-during-GC-after-row-delete artifacts). +//! Startup clears stale leases, validates the rollback checkpoint, then collects +//! obsolete snapshots and orphan directories before workers are admitted. use crate::commands::error::CommandError; use crate::ingress::inclusion_lane::dump_info::{self, delete_dump_dir}; -/// Run the five-step repair pass (see the module doc for the steps and -/// their ordering). +/// Repair interrupted artifact creation/collection before starting workers. pub(super) fn run_snapshot_hygiene( storage: &mut crate::storage::Storage, dumps_dir: &std::path::Path, ) -> Result<(), CommandError> { storage.reset_dump_leases()?; - require_finalized_snapshot(storage)?; - restamp_finalized_promotion(storage)?; + require_rollback_snapshot(storage)?; let gc_removed = snapshot_gc_at_startup(storage)?; let sweep_removed = sweep_orphan_dumps(storage, dumps_dir)?; tracing::debug!( @@ -51,36 +24,20 @@ pub(super) fn run_snapshot_hygiene( Ok(()) } -/// Require the finalized snapshot the lane will `from_dump` against. `setup` -/// registers the genesis snapshot and `run` gates on atomic setup completion, -/// so by the time the lane starts the snapshot must exist. A missing -/// one means the DB's setup is incomplete/corrupt — terminal -/// `SetupNotComplete` (re-run `setup`), not a cold-start to silently heal. -fn require_finalized_snapshot(storage: &mut crate::storage::Storage) -> Result<(), CommandError> { - if storage.finalized_dump()?.is_none() { - return Err(CommandError::Bootstrap( - crate::commands::error::BootstrapError::SetupNotComplete, - )); - } - Ok(()) -} - -/// Re-stamp `B` into the finalized dump's `info.toml` from the -/// authoritative DB row. Idempotent; closes the crash window between a -/// promotion's commit and the lane's in-place stamp. -fn restamp_finalized_promotion(storage: &mut crate::storage::Storage) -> Result<(), CommandError> { - if let Some(finalized) = storage.finalized_dump()? { - let path = finalized.dump.prefix; - dump_info::stamp_promoted_inclusion_block(&path, finalized.inclusion_block) - .map_err(|source| CommandError::ReferencedSnapshotArtifact { path, source })?; - } +fn require_rollback_snapshot(storage: &mut crate::storage::Storage) -> Result<(), CommandError> { + let snapshot = storage.rollback_snapshot()?.ok_or(CommandError::Bootstrap( + crate::commands::error::BootstrapError::SetupNotComplete, + ))?; + dump_info::read_info(&snapshot.dump.prefix).map_err(|source| { + CommandError::ReferencedSnapshotArtifact { + path: snapshot.dump.prefix, + source, + } + })?; Ok(()) } -/// Drop any dump rows that are now unreferenced (no pending, no -/// finalized, no leases). The companion `sweep_orphan_dumps` then -/// catches anything on disk that this leaves behind, plus -/// crash-during-create_dump orphans the SQLite layer never saw. +/// Delete obsolete rows before their files; the sweep retries leftover files. fn snapshot_gc_at_startup(storage: &mut crate::storage::Storage) -> Result { let removed = storage.gc_unreferenced_dumps()?; for row in &removed { @@ -105,7 +62,7 @@ fn snapshot_gc_at_startup(storage: &mut crate::storage::Storage) -> Result PreparedRuntime { let dumps_dir = std::path::Path::new(&run_config.data_dir).join("dumps"); std::fs::create_dir_all(&dumps_dir)?; - // Authority-neutral snapshot repair before the boundary; the five - // order-critical steps are documented in `startup_hygiene`. + // Validate the rollback artifact and collect obsolete snapshots before admission. super::startup_hygiene::run_snapshot_hygiene(&mut storage, &dumps_dir)?; // Prepare every remaining fallible or awaited dependency before the @@ -227,18 +226,13 @@ impl PreparedRuntime { ); detector.preflight_storage()?; - let tx_feed = L2TxFeed::new( - db_path.clone(), - shutdown.clone(), - L2TxFeedConfig::new(l1_config.identity.batch_submitter_address), - ); + let tx_feed = L2TxFeed::new(db_path.clone(), shutdown.clone(), L2TxFeedConfig::default()); // Configuration ends here: the remaining values are exactly what // `launch` hands to the workers, so the config structs never cross // the authority boundary. let lane_config = - InclusionLaneConfig::new(l1_config.identity.batch_submitter_address, dumps_dir) - .with_max_batch_open(run_config.max_batch_open()); + InclusionLaneConfig::new(dumps_dir).with_max_batch_open(run_config.max_batch_open()); let api_config = ApiConfig::new(domain, A::max_method_payload_bytes()); let listener = tokio::net::TcpListener::bind(&run_config.http_addr).await?; let bound_addr = listener.local_addr()?; @@ -911,7 +905,13 @@ mod tests { ) .expect("open storage"); storage - .insert_initial_finalized_dump(&finalized, 0, 0, 0, 0) + .complete_baseline_setup( + &finalized, + sequencer_core::history::ExecutedInputCount::ZERO, + 0, + 0, + false, + ) .expect("register finalized dump"); storage .append_safe_inputs(0, &[], submitter_address, &timing) @@ -919,7 +919,6 @@ mod tests { storage .initialize_open_state(0, crate::storage::SafeInputRange::empty_at(0)) .expect("initialize Tip"); - storage.complete_setup().expect("complete setup"); drop(storage); // One identity literal feeds both the reader and the L1 bundle, so diff --git a/sequencer/src/commands/setup/fill.rs b/sequencer/src/commands/setup/fill.rs index 5a4882db..c7a01996 100644 --- a/sequencer/src/commands/setup/fill.rs +++ b/sequencer/src/commands/setup/fill.rs @@ -1,19 +1,8 @@ // (c) Cartesi and individual authors (see AUTHORS) // SPDX-License-Identifier: Apache-2.0 (see LICENSE) -//! First-snapshot DB fills for the `setup` subcommand: the two ways `setup` -//! writes the initial finalized snapshot a freshly-prepared DB needs before -//! `run` will boot. -//! -//! - [`register_genesis_finalized_snapshot`] — plain `setup` (phase A): the -//! genesis app state at nonce 0 / cursor 0 / `B = 0`. -//! - [`fill_recovery_state`] — `setup --recovery`: the folded -//! cockroach-recovered state `S'`, tree anchored at `N'`, cursor past the -//! `<= C` directs already in `S'`. -//! -//! Both are setup-time, called once each from [`super::setup::setup`], and -//! never by a runtime worker — hence their own module, distinct from the -//! worker lifecycle in [`super::workers`]. +//! File-first baseline creation for setup. The complete baseline, optional +//! recovery root, and setup-complete fact become visible in one transaction. use crate::commands::error::{CommandError, SetupRecoveryError}; use crate::ingress::inclusion_lane::dump_info::{ @@ -21,185 +10,68 @@ use crate::ingress::inclusion_lane::dump_info::{ }; use sequencer_core::application::Application; -/// Register the genesis application state as the finalized snapshot. Called -/// once by `setup` (phase A). Idempotent: a re-run with a finalized snapshot -/// already present is a no-op, so the `initial_app` is dropped. -/// -/// The genesis dir is unique-per-attempt so a crash between dump creation and -/// `insert_finalized_dump` doesn't wedge a stale directory on the next -/// `setup`. The genesis checkpoint is born finalized: resume nonce 0, replay -/// cursor 0, `B` = 0 (implicit-genesis inclusion block, matching the row). -pub(crate) fn register_genesis_finalized_snapshot( +pub(crate) fn register_genesis_baseline( mut initial_app: A, storage: &mut crate::storage::Storage, dumps_dir: &std::path::Path, ) -> Result<(), CommandError> { - if storage.finalized_dump()?.is_some() { - return Ok(()); - } - // The violator here is a foreign `Application` impl supplied by the app - // crate, so a nonzero genesis boundary is a typed refusal with a - // diagnosis, not a panic across the crate boundary. It still runs - // first, before any write; nothing to unwind. - let genesis_count = initial_app.executed_input_count().get(); - if genesis_count != 0 { + let count = initial_app.executed_input_count(); + if count.get() != 0 { return Err(CommandError::AppBootstrap( sequencer_core::application::AppError::Internal { reason: format!( - "a genesis application must start at executed_input_count = 0, got {genesis_count}" + "a genesis application must start at executed_input_count = 0, got {}", + count.get() ), }, )); } - let nanos = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map(|d| d.as_nanos()) - .unwrap_or(0); - let genesis_dir = dumps_dir.join(format!("genesis-{nanos}")); - create_dump_dir_with_info( - &mut initial_app, - &genesis_dir, - &dump_info::DumpInfo { - format_version: dump_info::FORMAT_VERSION, - next_batch_nonce: 0, - l2_tx_index: 0, - promoted_inclusion_block: Some(0), - }, - ) - .map_err(|err| match err { - CreateDumpDirError::App(e) => CommandError::from(e), - CreateDumpDirError::Io(e) => CommandError::from(e), - })?; - storage.insert_initial_finalized_dump(&genesis_dir, 0, 0, 0, 0)?; + let prefix = write_baseline_dump(&mut initial_app, 0, dumps_dir)?; + storage.complete_baseline_setup(&prefix, count, 0, 0, false)?; Ok(()) } -/// Fill the freshly-wiped DB with the cockroach-recovered state, the recovery -/// analog of [`register_genesis_finalized_snapshot`]. Given -/// the folded application state `S'`, the resume batch nonce `N'`, and the -/// post-flush stopping block `C`, it leaves the DB in exactly the shape `run`'s -/// startup expects: a finalized snapshot of `S'`, a batch tree rooted at `N'`, -/// and a replay cursor past every `≤ C` direct (already executed inside `S'`). -/// -/// Pre-completion re-entry is **fail-loud**, not blindly idempotent. Only a -/// *completed* fill (finalized snapshot present — the last write) re-runs as a -/// safe no-op: its atomically bound snapshot and history base remain -/// authoritative even if the retry's newer fold reached a later count. Any -/// other existing root tip is a crashed mid-fill and is refused: -/// * a *different* `N'` (a different checkpoint, or the same one after `C` -/// advanced with more accepted `(B, C]` **batches**) would re-anchor the -/// tree while leaving the old root tip in place, silently breaking I16 — -/// [`SetupRecoveryError::PartialRecoveryMismatch`]; -/// * the *same* `N'` with no finalized snapshot is a crash between step 2 and -/// step 4. A re-sync may have advanced `C` with new **directs** (which leave -/// `N'` unchanged) that resuming would leave unsequenced, so the snapshot -/// cursor lags `S'` and `run` would drain + execute them a second time — -/// [`SetupRecoveryError::PartialRecoveryIncomplete`]. -/// -/// In both cases the operator wipes and re-runs (the one-shot recovery model). -/// -/// Order matters and differs from genesis: the root tip is opened *before* the -/// finalized snapshot, because the snapshot's `l2_tx_index` must equal the -/// global replay head *after* the tip's first frame has sequenced the `(*, C]` -/// directs. `run`'s catch-up replays `offset > l2_tx_index`, so those directs -/// (offsets below the head) are skipped — they are already in `S'`, while -/// on-chain `run`'s first batch (frame `safe_block ≥ C`) drains them once. +/// The terminal fold has already applied every direct through C. Its dump +/// is a local resume baseline; it is not a canonical comparison checkpoint. pub(crate) fn fill_recovery_state( mut recovered_app: A, resume_nonce: u64, - // `C`, the post-flush stop block; recorded as the snapshot's - // `promoted_inclusion_block` and the recovery tip frame's safe block. stop_block: u64, storage: &mut crate::storage::Storage, dumps_dir: &std::path::Path, ) -> Result<(), CommandError> { - // `K`: the absolute application boundary recovered by the canonical fold. - // It is deliberately independent of the replacement DB's physical replay - // cursor, which includes cursor-padding rows that must not execute again. - let base_executed_input_count = recovered_app.executed_input_count().get(); - - // Re-entry guard (rationale + the strict one-shot model are in the - // docstring). The tip is opened before the snapshot, so an existing root tip - // means a prior attempt: only a *completed* fill (finalized snapshot present) - // is a safe no-op; anything else is refused fail-loud. - if let Some(existing) = storage.open_tip_nonce()? { - // Different N' → re-anchoring would orphan the old root tip (I16 break). - if existing != resume_nonce { - return Err(SetupRecoveryError::PartialRecoveryMismatch { - existing_root_nonce: existing, - requested_nonce: resume_nonce, - } - .into()); - } - // Same N', no snapshot → crashed mid-fill; directs that landed since - // would be left unsequenced (cursor lags `S'` → `run` double-drains). - if storage.finalized_dump()?.is_none() { - return Err(SetupRecoveryError::PartialRecoveryIncomplete { - root_nonce: existing, - } - .into()); - } - return Ok(()); + if storage.is_setup_complete()? { + return Err(SetupRecoveryError::AlreadySetUp.into()); } - // No tip. A fresh fill — or anchor-only residue from a crash before step 2 — - // proceeds below (re-anchoring + opening the tip completes it). But a - // finalized snapshot with *no* root tip is residue from a different - // deployment mode: a plain `setup` that wrote the genesis snapshot and - // crashed before completion. A completed cockroach fill always has both - // (caught above), so reaching here with a snapshot means recovery is running - // over an un-wiped data dir — silently keeping it would mark setup complete - // over genesis instead of the folded `(S', N')`. Refuse (same fail-loud - // one-shot model as the partial-recovery guards above). - if let Some(finalized) = storage.finalized_dump()? { - return Err(SetupRecoveryError::RecoveryOverResidualSnapshot { - existing_finalized_block: finalized.inclusion_block, - } - .into()); - } - // 1. Anchor the tree at N' so the single parentless root carries it. - storage.set_batch_tree_anchor(resume_nonce)?; - // 2. Open the recovery root tip at N' (parentless, via the anchor), at frame - // safe block `C`, draining only the `≤ C` directs the fold folded into - // `S'` into its first frame's leading range — sequenced but not executed. - // This advances the next-undrained cursor past them so the resumed lane - // never re-leads them. Directs the resync pulled in past `C` (the resync - // runs to the live safe head `H1`, normally `> C`) stay undrained: `run`'s - // lane leads and executes them exactly once as the frontier advances - // `C → H1`. (Draining them here would skip them on catch-up while `S'` - // never executed them — a divergence.) - storage.open_recovery_tip(stop_block)?; - // 3. The finalized snapshot's replay cursor = the global valid replay head - // AFTER step 2's sequencing. - let head = storage.valid_ordered_l2_tx_head()?; - // The root's exclusive safe-input cursor is a separate durable floor. Its - // padding rows may later disappear from the valid view when standard - // recovery invalidates this root, but inputs already represented by S' - // must never become drainable again. - let base_safe_input_index = storage.next_undrained_safe_input_index()?; - // 4. Register S' as the finalized snapshot at block C (file-first). Unique - // per attempt so a crash before the DB row leaves only a swept orphan. + let count = recovered_app.executed_input_count(); + let prefix = write_baseline_dump(&mut recovered_app, resume_nonce, dumps_dir)?; + storage.complete_baseline_setup(&prefix, count, stop_block, resume_nonce, true)?; + Ok(()) +} + +fn write_baseline_dump( + app: &mut A, + next_batch_nonce: u64, + dumps_dir: &std::path::Path, +) -> Result { let nanos = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) - .map(|d| d.as_nanos()) - .unwrap_or(0); - let recovery_dir = dumps_dir.join(format!("recovery-{nanos}")); + .map_err(std::io::Error::other)? + .as_nanos(); + let prefix = dumps_dir.join(format!("baseline-{nanos}")); create_dump_dir_with_info( - &mut recovered_app, - &recovery_dir, - &dump_info::DumpInfo::at_recovery(resume_nonce, head, stop_block), + app, + &prefix, + &dump_info::DumpInfo { + format_version: dump_info::FORMAT_VERSION, + next_batch_nonce, + }, ) .map_err(|err| match err { CreateDumpDirError::App(e) => CommandError::from(e), CreateDumpDirError::Io(e) => CommandError::from(e), })?; - storage.insert_initial_finalized_dump( - &recovery_dir, - stop_block, - head, - base_executed_input_count, - base_safe_input_index, - )?; - Ok(()) + Ok(prefix) } #[cfg(test)] @@ -216,7 +88,7 @@ mod tests { AppError, AppOutputs, ApplicationProgress, ValidationOutcome, }; use sequencer_core::history::ExecutedInputCount; - use sequencer_core::l2_tx::{DirectInput, SequencedL2Tx, ValidUserOp}; + use sequencer_core::l2_tx::{DirectInput, ValidUserOp}; use sequencer_core::user_op::UserOp; use std::path::Path; use std::time::Duration; @@ -293,666 +165,278 @@ mod tests { } #[test] - fn recovery_binds_absolute_application_base_not_physical_cursor() { - use crate::storage::StoredSafeInput; + fn recovery_publishes_complete_baseline_without_application_padding() { use crate::storage::test_helpers::default_protocol_timing; - - let db = temp_db("fill-recovery-history-base"); - let mut storage = - Storage::initialize_for_command(db.path.as_str(), LifecycleCommand::Rebuild) - .expect("initialize rebuild"); - let dumps_dir = tempfile::tempdir().expect("dumps dir"); + use crate::storage::{FrontierMode, StoredSafeInput}; + let db = temp_db("complete-recovery-baseline"); + let mut storage = Storage::initialize_for_command(&db.path, LifecycleCommand::Rebuild) + .expect("initialize rebuild"); + let dumps = tempfile::tempdir().expect("dumps"); let submitter = Address::repeat_byte(0x99); - let direct = Address::repeat_byte(0x22); + pin_test_deployment_identity(&mut storage, submitter); storage - .append_safe_inputs( - 100, + .append_safe_inputs_with_timestamp( + 150, + 150, &[ StoredSafeInput { - sender: direct, - payload: vec![0x01], - block_number: 20, + sender: Address::repeat_byte(0x22), + payload: vec![1], + block_number: 100, }, StoredSafeInput { - sender: direct, - payload: vec![0x02], - block_number: 30, + sender: Address::repeat_byte(0x22), + payload: vec![2], + block_number: 120, }, ], submitter, &default_protocol_timing(), + FrontierMode::DeferUntilAnchorSet, ) - .expect("sync recovered inputs"); - + .expect("ingest prefix and later direct"); fill_recovery_state( CountedSweepTestApp::new(41), 3, 100, &mut storage, - dumps_dir.path(), + dumps.path(), ) - .expect("fill recovered state"); - + .expect("complete baseline"); + assert!(storage.is_setup_complete().expect("completion")); + assert_eq!(storage.batch_tree_anchor().expect("anchor"), 3); + let history = storage.history_state().expect("history"); + assert_eq!(history.base_executed_input_count, 41); + assert_eq!(history.base_safe_block, 100); + let root = storage.open_state().expect("root").expect("open root"); + assert_eq!(root.safe_block, 100); assert_eq!( - storage - .history_state() - .expect("history") - .base_executed_input_count, - Some(41), - "K comes from the recovered Application state" + storage.next_executed_input_count().expect("head"), + ExecutedInputCount::new(41) ); - assert_eq!( + assert!( storage - .history_state() - .expect("history") - .base_safe_input_index, - Some(2), - "the recovery root's exclusive safe-input cursor becomes the durable drain floor" + .finalized_dump() + .expect("accepted checkpoint") + .is_none() ); - let finalized = storage - .finalized_dump() - .expect("read finalized") - .expect("finalized snapshot"); + let snapshot = storage + .latest_snapshot() + .expect("baseline") + .expect("baseline"); + assert_eq!(snapshot.executed_input_count, ExecutedInputCount::new(41)); + let restored = + CountedSweepTestApp::from_dump(&dump_info::app_prefix(&snapshot.dump.prefix)) + .expect("restore baseline"); assert_eq!( - finalized.l2_tx_index, 2, - "the replacement DB cursor can differ from absolute application count K" + restored.executed_input_count(), + snapshot.executed_input_count ); assert_eq!( - CountedSweepTestApp::from_dump(&dump_info::app_prefix(&finalized.dump.prefix)) - .expect("reload counted snapshot") - .executed_input_count() - .get(), - 41, - "the snapshot bytes and durable K establish the same application boundary" + storage + .read(|tx| tx + .query_row("SELECT COUNT(*) FROM application_inputs", [], |row| row + .get::<_, i64>(0))) + .expect("application rows"), + 0 ); - - // Model a retry whose re-sync/fold advanced through more direct inputs - // without changing N'. The completed snapshot/base pair is already the - // durable boundary; the later fold must not reinterpret this era. - fill_recovery_state( + let err = fill_recovery_state( CountedSweepTestApp::new(42), - 3, - 100, + 4, + 150, &mut storage, - dumps_dir.path(), + dumps.path(), ) - .expect("a completed fill remains authoritative across a later fold"); - assert_eq!( - storage - .history_state() - .expect("preserved history") - .base_executed_input_count, - Some(41), - "a completed retry preserves the snapshot-bound K" - ); + .expect_err("completed rebuild is one-shot"); + assert!(matches!( + err, + CommandError::Bootstrap(crate::commands::error::BootstrapError::SetupRecovery( + SetupRecoveryError::AlreadySetUp + )) + )); assert_eq!( - storage - .history_state() - .expect("preserved history") - .base_safe_input_index, - Some(2), - "a completed retry preserves the snapshot-bound drain floor" + storage.history_state().expect("unchanged baseline"), + history ); } - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn standard_recovery_never_redrains_below_cockroach_floor() { - use crate::storage::StoredSafeInput; - use crate::storage::test_helpers::default_protocol_timing; - - let db = temp_db("cockroach-drain-floor"); - let dumps_dir = tempfile::tempdir().expect("dumps dir"); - let timing = default_protocol_timing(); - let submitter = Address::repeat_byte(0x99); - let direct = Address::repeat_byte(0x22); - let mut storage = - Storage::initialize_for_command(db.path.as_str(), LifecycleCommand::Rebuild) - .expect("initialize rebuild"); - pin_test_deployment_identity(&mut storage, submitter); + #[test] + fn failed_baseline_registration_rolls_back_root_history_and_completion() { + let db = temp_db("baseline-registration-rollback"); + let mut storage = Storage::initialize_for_command(&db.path, LifecycleCommand::Rebuild) + .expect("initialize rebuild"); + let dumps = tempfile::tempdir().expect("dumps"); storage - .append_safe_inputs( - 100, - &[ - StoredSafeInput { - sender: direct, - payload: vec![0x01], - block_number: 20, - }, - StoredSafeInput { - sender: direct, - payload: vec![0x02], - block_number: 30, - }, - ], - submitter, - &timing, - ) - .expect("sync recovered inputs"); - - // S' already includes these two directs. The root sequences them only - // as physical replay padding, and binds their exclusive cursor as the - // durable floor beside the snapshot. - fill_recovery_state( + .write(|tx| { + tx.execute_batch( + "CREATE TRIGGER fail_baseline BEFORE INSERT ON snapshots + BEGIN SELECT RAISE(ABORT, 'injected baseline failure'); END;", + ) + }) + .expect("inject failure"); + let err = fill_recovery_state( CountedSweepTestApp::new(41), 3, 100, &mut storage, - dumps_dir.path(), + dumps.path(), ) - .expect("fill recovered state"); - storage.complete_setup().expect("complete rebuild"); - assert_eq!( - storage - .history_state() - .expect("history") - .base_safe_input_index, - Some(2) - ); - - // Age the recovery root into standard recovery. Invalidating it removes - // its padding rows from the valid view; the replacement Tip must still - // begin at the durable floor rather than re-sequencing indices 0 and 1. - storage - .append_safe_inputs( - 1_500, - &[StoredSafeInput { - sender: direct, - payload: vec![0x03], - block_number: 1_400, - }], - submitter, - &timing, - ) - .expect("advance safe head with one post-floor direct"); - assert_eq!( - storage.recover_post_flush(1_200).expect("recover root"), - vec![0] - ); + .expect_err("registration must fail"); + assert!(err.to_string().contains("injected baseline failure")); + assert!(!storage.is_setup_complete().expect("completion absent")); + assert!(storage.open_state().expect("root absent").is_none()); + assert_eq!(storage.batch_tree_anchor().expect("unchanged anchor"), 0); assert_eq!( storage - .next_undrained_safe_input_index() - .expect("post-recovery cursor"), - 3 - ); - let replay = storage - .ordered_l2_txs_page_from(0, 16) - .expect("valid replay"); - assert_eq!( - replay.len(), - 1, - "only the post-floor direct belongs to replacement history" + .read( + |tx| tx.query_row("SELECT COUNT(*) FROM history_state", [], |row| row + .get::<_, i64>(0)) + ) + .expect("no history"), + 0 ); - let row = &replay[0]; assert_eq!( - row.executed_input_offset, - Some(ExecutedInputCount::new(41)), - "the first post-floor direct must reuse the recovered application boundary K" - ); - match &row.tx { - SequencedL2Tx::Direct(input) => assert_eq!(input.payload, [0x03]), - SequencedL2Tx::UserOp(_) => panic!("expected the post-floor direct"), - } - drop(storage); - - // Restart the real lane from S'. Catch-up must execute only the - // post-floor direct at offset 41, advancing the application to 42. If - // the invalidated padding were re-sequenced, the count would be larger. - // Force an empty batch close so the post-catch-up state is visible. - let storage = Storage::open(db.path.as_str()).expect("reopen for lane"); - let config = InclusionLaneConfig { - batch_submitter_address: submitter, - dumps_dir: dumps_dir.path().to_path_buf(), - max_user_ops_per_chunk: 16, - safe_input_buffer_capacity: 16, - max_batch_open: Duration::from_millis(10), - idle_poll_interval: Duration::from_millis(2), - frontier_min_interval: Duration::ZERO, - }; - let shutdown = RuntimeScope::default(); - let (_tx, handle) = - InclusionLane::::start(16, shutdown.clone(), storage, config); - - let advanced_once = wait_until(Duration::from_secs(5), || { - let mut observer = Storage::open(db.path.as_str()).expect("open observer"); - let Some(pending) = observer.latest_pending_dump().expect("read pending") else { - return false; - }; - CountedSweepTestApp::from_dump(&dump_info::app_prefix(&pending.dump.prefix)) - .expect("load post-catch-up snapshot") - .executed_input_count() - .get() - == 42 - }) - .await; - assert!( - advanced_once, - "lane catch-up must execute the post-floor direct exactly once" + std::fs::read_dir(dumps.path()) + .expect("orphan dump") + .count(), + 1 ); - - shutdown_lane(&shutdown, handle).await; - } - - #[test] - fn plain_setup_refuses_a_nonzero_genesis_application_boundary() { - let db = temp_db("genesis-history-base"); - let mut storage = - Storage::initialize_for_command(db.path.as_str(), LifecycleCommand::Setup) - .expect("initialize setup"); - let dumps_dir = tempfile::tempdir().expect("dumps dir"); - - let result = register_genesis_finalized_snapshot( - CountedSweepTestApp::new(1), + storage + .write(|tx| tx.execute_batch("DROP TRIGGER fail_baseline")) + .expect("remove failure"); + fill_recovery_state( + CountedSweepTestApp::new(42), + 4, + 150, &mut storage, - dumps_dir.path(), - ); - assert!( - matches!( - result, - Err(CommandError::AppBootstrap( - sequencer_core::application::AppError::Internal { .. } - )) - ), - "genesis must begin at application count zero, got: {result:?}" - ); - assert!(storage.finalized_dump().expect("read finalized").is_none()); + dumps.path(), + ) + .expect("fresh retry after atomic rollback"); assert_eq!( storage .history_state() .expect("history") .base_executed_input_count, - Some(0) - ); - assert_eq!( - storage - .history_state() - .expect("history") - .base_safe_input_index, - Some(0) + 42 ); + assert_eq!(storage.open_tip_nonce().expect("root nonce"), Some(4)); } #[test] - fn fill_recovery_state_roots_tree_at_n_prime_and_skips_pre_executed_directs() { - use crate::storage::StoredSafeInput; - use crate::storage::test_helpers::default_protocol_timing; - - let db = temp_db("fill-recovery"); - let mut storage = - Storage::initialize_for_command(db.path.as_str(), LifecycleCommand::Rebuild) - .expect("initialize rebuild"); - let dumps_dir = tempfile::tempdir().expect("dumps dir"); - let submitter = alloy_primitives::Address::repeat_byte(0x99); - let direct = alloy_primitives::Address::repeat_byte(0x22); - let timing = default_protocol_timing(); - - // Sync a safe head at C = 100 with three `≤ C` inputs: a batch (from the - // submitter) and two directs. These stand in for the (A, C] stream the - // fold already folded into S'. - let inputs = vec![ - StoredSafeInput { - sender: submitter, - payload: vec![0x00], - block_number: 10, - }, - StoredSafeInput { - sender: direct, - payload: vec![0xAA], - block_number: 20, - }, - StoredSafeInput { - sender: direct, - payload: vec![0xBB], - block_number: 30, - }, - ]; - storage - .append_safe_inputs(100, &inputs, submitter, &timing) - .expect("sync through C"); - - // Fill at resume nonce N' = 3, C = 100. - let n_prime = 3; - fill_recovery_state(SweepTestApp, n_prime, 100, &mut storage, dumps_dir.path()) - .expect("recovery fill"); - - // The tree is anchored at N' and the single root tip carries it. - assert_eq!(storage.batch_tree_anchor().expect("anchor"), n_prime); - let root_idx = storage - .latest_batch_index() - .expect("idx") - .expect("a root tip exists"); - assert_eq!( - storage.batch_nonce(root_idx).expect("nonce"), - n_prime, - "root tip roots at N' (via the anchor)" + fn plain_setup_refuses_a_nonzero_genesis_application_boundary() { + let db = temp_db("nonzero-genesis"); + let mut storage = Storage::initialize_for_command(&db.path, LifecycleCommand::Setup) + .expect("initialize setup"); + let dumps = tempfile::tempdir().expect("dumps"); + assert!( + register_genesis_baseline(CountedSweepTestApp::new(1), &mut storage, dumps.path()) + .is_err() ); - - // All three `≤ C` inputs are sequenced into the root frame, so the - // next-undrained cursor sits past them — the resumed lane never re-leads - // them, and they are already in S'. - assert_eq!( - storage.next_undrained_safe_input_index().expect("cursor"), - 3 + assert!(!storage.is_setup_complete().expect("completion absent")); + assert!( + storage + .latest_snapshot() + .expect("snapshot absent") + .is_none() ); - - // Finalized snapshot at C, replay cursor = the post-sequencing head, so - // run's catch-up (offset > l2_tx_index) skips the pre-executed directs. - let finalized = storage - .finalized_dump() - .expect("read") - .expect("finalized snapshot exists"); - assert_eq!(finalized.inclusion_block, 100); assert_eq!( - finalized.l2_tx_index, 3, - "catch-up starts past the pre-executed (≤C) directs" + std::fs::read_dir(dumps.path()) + .expect("no artifacts") + .count(), + 0 ); - - // Idempotent: a re-run (crash before setup completion) is a no-op, not a - // duplicate-insert error. - fill_recovery_state(SweepTestApp, n_prime, 100, &mut storage, dumps_dir.path()) - .expect("re-run is idempotent"); - assert_eq!(storage.batch_tree_anchor().expect("anchor"), n_prime); } #[test] - fn fill_recovery_state_leaves_post_c_directs_undrained() { - // The post-flush resync runs to the live safe head H1, normally > the - // checkpoint stop block C. The fold folds only `<= C` into S'; directs in - // (C, H1] must stay UNDRAINED so run leads + executes them exactly once. - // Draining them here (the old behavior) would skip them on catch-up while - // S' never executed them — a vanished deposit / divergence. - use crate::storage::StoredSafeInput; + fn root_invalidation_keeps_baseline_floor_and_replays_only_post_baseline_directs() { use crate::storage::test_helpers::default_protocol_timing; - - let db = temp_db("fill-recovery-post-c"); - let mut storage = - Storage::initialize_for_command(db.path.as_str(), LifecycleCommand::Rebuild) - .expect("initialize rebuild"); - let dumps_dir = tempfile::tempdir().expect("dumps dir"); - let submitter = alloy_primitives::Address::repeat_byte(0x99); - let direct = alloy_primitives::Address::repeat_byte(0x22); - let timing = default_protocol_timing(); - - // C = 100; the resync reached H1 = 150. Three directs <= C, one at block - // 120 in the (C, H1] window. - let inputs = vec![ - StoredSafeInput { - sender: direct, - payload: vec![0x01], - block_number: 10, - }, - StoredSafeInput { - sender: direct, - payload: vec![0x02], - block_number: 20, - }, - StoredSafeInput { - sender: direct, - payload: vec![0x03], - block_number: 30, - }, - StoredSafeInput { - sender: direct, - payload: vec![0x04], - block_number: 120, - }, - ]; + use crate::storage::{FrontierMode, StoredSafeInput}; + use sequencer_core::history::HistoryClaim; + let db = temp_db("baseline-survives-root-invalidation"); + let mut storage = Storage::initialize_for_command(&db.path, LifecycleCommand::Rebuild) + .expect("initialize rebuild"); + let dumps = tempfile::tempdir().expect("dumps"); + let submitter = Address::repeat_byte(0x99); + pin_test_deployment_identity(&mut storage, submitter); storage - .append_safe_inputs(150, &inputs, submitter, &timing) - .expect("sync through H1"); - - fill_recovery_state(SweepTestApp, 3, 100, &mut storage, dumps_dir.path()).expect("fill"); - - // All four inputs are in safe_inputs ... - assert_eq!(storage.safe_input_end_exclusive().expect("end"), 4); - // ... but only the three <= C directs are sequenced; the block-120 direct - // (index 3) stays undrained for run to lead + execute. - assert_eq!( - storage.next_undrained_safe_input_index().expect("cursor"), + .append_safe_inputs_with_timestamp( + 1500, + 1500, + &[ + StoredSafeInput { + sender: Address::repeat_byte(0x22), + payload: vec![1], + block_number: 100, + }, + StoredSafeInput { + sender: Address::repeat_byte(0x22), + payload: vec![2], + block_number: 1400, + }, + ], + submitter, + &default_protocol_timing(), + FrontierMode::DeferUntilAnchorSet, + ) + .expect("ingest inputs"); + fill_recovery_state( + CountedSweepTestApp::new(41), 3, - "the (C, H1] direct must remain undrained" - ); - let finalized = storage - .finalized_dump() - .expect("read") - .expect("finalized snapshot"); + 100, + &mut storage, + dumps.path(), + ) + .expect("complete baseline"); + let history = storage.history_state().expect("history"); assert_eq!( - finalized.l2_tx_index, 3, - "catch-up cursor stops at the <= C directs, not the (C, H1] one" + storage.recover_aging_tip(1200).expect("recover root"), + vec![0] ); - } - - #[test] - fn fill_recovery_state_drain_boundary_is_inclusive_at_exactly_c() { - // The (C, H1] split must be inclusive at exactly C: a direct at block C - // is folded into S' and drained here; one at C+1 is not. Off-by-one in - // either direction is the same loss/double-execution class as the (C, H1] - // bug, from the boundary. - use crate::storage::StoredSafeInput; - use crate::storage::test_helpers::default_protocol_timing; - - let db = temp_db("fill-recovery-boundary"); - let mut storage = - Storage::initialize_for_command(db.path.as_str(), LifecycleCommand::Rebuild) - .expect("initialize rebuild"); - let dumps_dir = tempfile::tempdir().expect("dumps dir"); - let submitter = alloy_primitives::Address::repeat_byte(0x99); - let direct = alloy_primitives::Address::repeat_byte(0x22); - let timing = default_protocol_timing(); - - // C = 100. Directs straddling it exactly: block 99 (< C), block 100 - // (== C), block 101 (> C). Synced head H1 = 150. - let inputs = vec![ - StoredSafeInput { - sender: direct, - payload: vec![0x01], - block_number: 99, - }, - StoredSafeInput { - sender: direct, - payload: vec![0x02], - block_number: 100, - }, - StoredSafeInput { - sender: direct, - payload: vec![0x03], - block_number: 101, - }, - ]; - storage - .append_safe_inputs(150, &inputs, submitter, &timing) - .expect("sync through H1"); - - fill_recovery_state(SweepTestApp, 0, 100, &mut storage, dumps_dir.path()).expect("fill"); - - // Blocks 99 and 100 (<= C) are drained; block 101 (> C) is not. Inclusive - // at exactly C: cursor sits at index 2, not 1 (would drop the ==C direct) - // and not 3 (would over-drain the >C direct). + let current = storage.history_state().expect("current history"); + assert_eq!(current.base_safe_block, 100); + assert_eq!(current.base_executed_input_count, 41); assert_eq!( - storage.next_undrained_safe_input_index().expect("cursor"), - 2, - "the block-C direct must be drained; the block-C+1 direct must not" - ); - let finalized = storage - .finalized_dump() - .expect("read") - .expect("finalized snapshot"); - assert_eq!(finalized.l2_tx_index, 2); - } - - #[test] - fn fill_recovery_state_refuses_re_run_with_a_different_nonce() { - // A re-run with a *different* resume nonce (e.g. a different - // checkpoint, or the same one after C advanced) would move the anchor - // while leaving the old root tip — a silent I16 break. It must fail loud. - use crate::storage::StoredSafeInput; - use crate::storage::test_helpers::default_protocol_timing; - - let db = temp_db("fill-recovery-nonce-mismatch"); - let mut storage = - Storage::initialize_for_command(db.path.as_str(), LifecycleCommand::Rebuild) - .expect("initialize rebuild"); - let dumps_dir = tempfile::tempdir().expect("dumps dir"); - let submitter = alloy_primitives::Address::repeat_byte(0x99); - let timing = default_protocol_timing(); - let inputs = vec![StoredSafeInput { - sender: alloy_primitives::Address::repeat_byte(0x22), - payload: vec![0xAA], - block_number: 10, - }]; - storage - .append_safe_inputs(100, &inputs, submitter, &timing) - .expect("sync"); - - // First attempt fills at N' = 3 (opens a root tip carrying nonce 3). - fill_recovery_state(SweepTestApp, 3, 100, &mut storage, dumps_dir.path()) - .expect("first fill"); - - // Re-run with a DIFFERENT nonce (5): the existing root tip carries 3, so - // the tip-nonce guard fires *before* the finalized short-circuit. - let err = fill_recovery_state(SweepTestApp, 5, 100, &mut storage, dumps_dir.path()) - .expect_err("different-nonce re-run must fail loud"); - assert!( - matches!( - err, - CommandError::Bootstrap(crate::commands::error::BootstrapError::SetupRecovery( - SetupRecoveryError::PartialRecoveryMismatch { - existing_root_nonce: 3, - requested_nonce: 5, - } - )) - ), - "expected PartialRecoveryMismatch, got {err:?}" - ); - } - - #[test] - fn fill_recovery_state_refuses_incomplete_same_nonce_re_run() { - // A crash between opening the recovery root tip (fill step 2) and - // writing the finalized snapshot (step 4) leaves "tip exists, no - // finalized". A same-N' re-run must NOT resume — directs that landed - // since would be left unsequenced, leaving the snapshot cursor behind - // `S'` and double-draining them on `run`. Must fail loud. - // - // We reproduce the EXACT on-disk residue a crashed fill leaves by - // running the real fill writes (anchor + `open_recovery_tip`) and - // stopping before the snapshot — no process crash / test hook needed, - // since the residue is fully determined by which committed storage - // writes happened. - use crate::storage::StoredSafeInput; - use crate::storage::test_helpers::default_protocol_timing; - - let db = temp_db("fill-recovery-incomplete"); - let mut storage = - Storage::initialize_for_command(db.path.as_str(), LifecycleCommand::Rebuild) - .expect("initialize rebuild"); - let dumps_dir = tempfile::tempdir().expect("dumps dir"); - let submitter = alloy_primitives::Address::repeat_byte(0x99); - let direct = alloy_primitives::Address::repeat_byte(0x22); - let timing = default_protocol_timing(); - - // Sync a safe head with one `≤ C` direct, then reproduce a crashed - // mid-fill at N' = 3 with the production writes: anchor set + recovery - // tip opened at C = 100, but no finalized snapshot. - storage - .append_safe_inputs( - 100, - &[StoredSafeInput { - sender: direct, - payload: vec![0xAA], - block_number: 20, - }], - submitter, - &timing, - ) - .expect("sync"); - storage.set_batch_tree_anchor(3).expect("anchor"); - storage - .open_recovery_tip(100) - .expect("open recovery root tip"); - assert!( - storage.finalized_dump().expect("read").is_none(), - "precondition: no finalized snapshot (crashed mid-fill)" + current.version.recovery_generation.get(), + history.version.recovery_generation.get() + 1 ); - - // A new direct lands during the retry gap (C advances; N' unchanged). - storage - .append_safe_inputs( - 130, - &[StoredSafeInput { - sender: direct, - payload: vec![0xBB], - block_number: 120, - }], - submitter, - &timing, + let snapshot = storage + .latest_snapshot() + .expect("baseline") + .expect("baseline survives"); + let mut app = CountedSweepTestApp::from_dump(&dump_info::app_prefix(&snapshot.dump.prefix)) + .expect("restore"); + let page = storage + .canonical_history_page( + HistoryClaim { + version: current.version, + next_input: snapshot.executed_input_count, + }, + 16, ) - .expect("resync with a new direct"); - - // Re-run at the SAME N' = 3 must refuse — resuming would leave the new - // direct unsequenced → double-execution on `run`. - let err = fill_recovery_state(SweepTestApp, 3, 130, &mut storage, dumps_dir.path()) - .expect_err("incomplete same-nonce re-run must fail loud"); - assert!( - matches!( - err, - CommandError::Bootstrap(crate::commands::error::BootstrapError::SetupRecovery( - SetupRecoveryError::PartialRecoveryIncomplete { root_nonce: 3 } - )) - ), - "expected PartialRecoveryIncomplete, got {err:?}" - ); - } - - #[test] - fn fill_recovery_state_refuses_over_residual_finalized_snapshot() { - // `setup --recovery` over an un-wiped data dir left by a plain - // `setup` that wrote the genesis finalized snapshot and crashed before - // completion (finalized snapshot present, NO root tip). A completed - // cockroach fill always has both, so this residue must fail loud — - // silently keeping it would mark setup complete over genesis instead of - // the folded `(S', N')`. - let db = temp_db("fill-recovery-residue"); - let mut storage = - Storage::initialize_for_command(db.path.as_str(), LifecycleCommand::Setup) - .expect("initialize plain setup residue"); - let dumps_dir = tempfile::tempdir().expect("dumps dir"); - - // Plain-setup residue: genesis finalized snapshot, no root tip. - register_genesis_finalized_snapshot(SweepTestApp, &mut storage, dumps_dir.path()) - .expect("genesis snapshot"); + .expect("replacement history"); + assert_eq!(page.rows.len(), 1); + assert_eq!(page.rows[0].offset, ExecutedInputCount::new(41)); + match &page.rows[0].context { + crate::storage::L2TxContext::DirectInput { tx, .. } => { + assert_eq!(tx.block_number, 1400); + sequencer_core::application::execute_direct_input(&mut app, tx) + .expect("replay direct"); + } + _ => panic!("expected post-baseline direct"), + } + assert_eq!(app.executed_input_count(), ExecutedInputCount::new(42)); assert!( - storage.latest_batch_index().expect("idx").is_none(), - "precondition: no root tip" + storage + .recover_aging_tip(1200) + .expect("fresh replacement") + .is_empty() ); - - let err = fill_recovery_state(SweepTestApp, 3, 100, &mut storage, dumps_dir.path()) - .expect_err("recovery over residual finalized snapshot must fail loud"); - assert!( - matches!( - err, - CommandError::Bootstrap(crate::commands::error::BootstrapError::SetupRecovery( - SetupRecoveryError::RecoveryOverResidualSnapshot { - existing_finalized_block: 0, - } - )) - ), - "expected RecoveryOverResidualSnapshot, got {err:?}" + assert_eq!( + storage.next_executed_input_count().expect("head"), + ExecutedInputCount::new(42) ); } - /// The (C, H1] deposit-window property, end-to-end at the run side: a portal - /// deposit that lands AFTER the flush fixed `C` but within the resync's reach - /// (`H1 > C`) is left UNDRAINED by the recovery fill, then led + executed - /// EXACTLY ONCE by the real inclusion lane as the safe frontier advances - /// `C -> H1` — not lost (drained early / skipped on catch-up) and not - /// double-credited (executed by both the recovery drain and the lane lead). - /// - /// This drives the production lane in-process against a recovered DB — no - /// subprocess, no L1, no test hooks — and replaces the former subprocess - /// e2e. The per-cap unit tests (`..._leaves_post_c_directs_undrained`, - /// `..._drain_boundary_is_inclusive_at_exactly_c`, the fold-source bound) - /// pin each piece; this is the only test that observes the *composition* - /// crediting the deposit exactly once across the setup -> run handoff. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn recovery_then_lane_credits_post_c_deposit_exactly_once() { use crate::storage::StoredSafeInput; @@ -1028,13 +512,9 @@ mod tests { "the (C, H1] deposit must be left UNDRAINED at index 3" ); let finalized = storage - .finalized_dump() + .latest_snapshot() .expect("read") - .expect("finalized S' exists"); - assert_eq!( - finalized.l2_tx_index, 3, - "catch-up starts past the <= C directs" - ); + .expect("baseline exists"); let s_prime = WalletApp::from_dump(&dump_info::app_prefix(&finalized.dump.prefix)) .expect("load S'"); assert_eq!( @@ -1052,7 +532,6 @@ mod tests { // input; the short `max_batch_open` forces a batch-close snapshot. let storage = Storage::open(db.path.as_str()).expect("reopen for lane"); let config = InclusionLaneConfig { - batch_submitter_address: submitter, dumps_dir: dumps_dir.path().to_path_buf(), max_user_ops_per_chunk: 16, safe_input_buffer_capacity: 16, @@ -1069,7 +548,7 @@ mod tests { // the lane writes at batch close. Wait for it to reflect the credit. let credited = wait_until(Duration::from_secs(5), || { let mut s = Storage::open(db.path.as_str()).expect("open observer"); - match s.latest_pending_dump().expect("read pending") { + match s.latest_snapshot().expect("read pending") { Some(p) => { WalletApp::from_dump(&dump_info::app_prefix(&p.dump.prefix)) .expect("load lane snapshot") @@ -1096,7 +575,7 @@ mod tests { "the deposit's drain cursor must advance exactly one past it" ); let dump = s - .latest_pending_dump() + .latest_snapshot() .expect("pending") .expect("a post-deposit pending dump"); let app = diff --git a/sequencer/src/commands/setup/mod.rs b/sequencer/src/commands/setup/mod.rs index 20ed23a5..a6725e70 100644 --- a/sequencer/src/commands/setup/mod.rs +++ b/sequencer/src/commands/setup/mod.rs @@ -17,9 +17,9 @@ //! **address**, and fee-oracle identity), and persist the first price. //! `setup` never signs. //! 4. Initial L1 sync: read all direct inputs up to the current safe head. -//! 5. For plain setup, construct and register the genesis application state as -//! the finalized snapshot. Recovery supplies its state from the checkpoint. -//! 6. Commit the `setup_complete` fact. +//! 5. Create the durable genesis or recovered application baseline dump. +//! 6. Atomically register complete baseline metadata, its artifact, any recovery +//! root, and the `setup_complete` fact. //! //! `setup` is L1-read-only: it takes the batch-submitter address (not the //! key) and does no L1 writes. @@ -231,7 +231,7 @@ where // block (the scan genesis — no input exists before it). // `B = 0` is the genesis bootstrap (no checkpoint) and is always valid. // (plain setup detects only; loading a non-genesis checkpoint machine and - // the `A < B` check are `setup --recovery`'s job.) + // the checkpoint clock check are `setup --recovery`'s job.) if config.checkpoint_block != 0 && config.checkpoint_block < input_reader.app_deployment_block() { return Err(BootstrapError::CheckpointBeforeAppDeployment { @@ -318,27 +318,15 @@ where nonce_views, )?; - // Refuse to register genesis over leftover recovery state. A - // `setup --recovery` that crashed before completion leaves a non-zero - // batch-tree anchor (and maybe a root tip); booting genesis-style over - // it would root the tree at the recovery nonce instead of 0. Fail loud - // Setup completion is absent in both the fresh-genesis and interrupted - // recovery cases, so we check the anchor explicitly. Operator wipes - // the data dir and re-runs. - let anchor = storage.batch_tree_anchor()?; - if anchor != 0 { - return Err(SetupRecoveryError::GenesisOverRecoveryResidue { anchor }.into()); - } - // ── Genesis snapshot ───────────────────────────────────── // Construct only after the admission facts and every // detect-and-refuse gate. Factory errors follow normal setup settlement; // completed no-ops and recovery never construct genesis state. let genesis_app = genesis_app()?; - fill::register_genesis_finalized_snapshot::(genesis_app, &mut storage, &dumps_dir)?; + fill::register_genesis_baseline::(genesis_app, &mut storage, &dumps_dir)?; } - // The caller commits setup_complete + Ready as one final transaction. + // Baseline registration committed setup_complete with the restore point. tracing::info!( data_dir = %config.data_dir, chain_id = identity.chain_id, @@ -392,10 +380,13 @@ fn settle_setup_lifecycle( ) -> Result<(), CommandError> { match result { Ok(()) => { - // The completion fact is part of the command, not telemetry: if - // it cannot be written, setup did not complete. - let mut storage = storage::Storage::open_writer(db_path)?; - storage.complete_setup()?; + let storage = storage::Storage::open_read_only(db_path)?; + if !storage.is_setup_complete()? { + return Err(storage::LifecycleError::Malformed( + "setup returned without its complete baseline".to_string(), + ) + .into()); + } Ok(()) } Err(_) => { @@ -426,17 +417,33 @@ struct Checkpoint { impl Checkpoint { /// Load `S` from the dump dir, derive `A` and `N`, and enforce the load-time - /// precondition `A < B` (else the `(A, B]` fridge range is ill-defined). All - /// failures are terminal — the operator must supply a valid checkpoint. + /// precondition `A < B`, except for the known empty genesis checkpoint. + /// Equality otherwise hides pending directs later in the same block. fn load(dir: &std::path::Path, checkpoint_block: u64) -> Result { let load_err = |message: String| SetupRecoveryError::CheckpointLoad { path: dir.display().to_string(), message, }; let info = dump_info::read_info(dir).map_err(|e| load_err(e.to_string()))?; + let checkpoint = + dump_info::read_checkpoint_info(dir).map_err(|e| load_err(e.to_string()))?; + if checkpoint.inclusion_block != checkpoint_block { + return Err(load_err(format!( + "checkpoint receipt block {} differs from configured block {checkpoint_block}", + checkpoint.inclusion_block, + ))); + } + if checkpoint.next_batch_nonce != info.next_batch_nonce { + return Err(load_err( + "checkpoint receipt nonce differs from immutable dump metadata".into(), + )); + } let app = A::from_dump(&dump_info::app_prefix(dir)).map_err(|e| load_err(e.to_string()))?; let executed_safe_block = app.last_executed_safe_block(); - if executed_safe_block >= checkpoint_block { + let is_genesis = checkpoint_block == 0 + && info.next_batch_nonce == 0 + && app.executed_input_count() == sequencer_core::history::ExecutedInputCount::ZERO; + if executed_safe_block >= checkpoint_block && !is_genesis { return Err(SetupRecoveryError::CheckpointNotBeforeBlock { executed_safe_block, checkpoint_block, @@ -511,7 +518,7 @@ async fn recover( where A: Application + 'static, { - // 1. Load the trusted checkpoint (S, A, N, B); require A < B. + // 1. Load the trusted checkpoint (S, A, N, B); require A < B or empty genesis. let checkpoint_dir = std::path::Path::new( config .checkpoint_dump_dir @@ -564,10 +571,8 @@ where stop_block, )?; - // 6. Fill the DB: finalized S', tree anchored at N', cursor past the ≤C - // directs (already in S'). run boots from this state, and its first sync - // populates the gold frontier from L1 with the anchor = N' (so the folded - // `< N'` batches are skipped as trusted collapsed history, not foreign). + // 6. Publish the local resume baseline and root atomically. The first run + // scans acceptance only after C, beginning with expected nonce N'. fill::fill_recovery_state(recovered_app, resume_nonce, stop_block, storage, dumps_dir)?; tracing::info!( @@ -757,6 +762,144 @@ mod tests { use crate::storage::StoredSafeInput; use crate::storage::test_helpers::{SENDER_A, default_protocol_timing, temp_db}; + #[test] + fn checkpoint_load_requires_an_export_receipt_matching_the_artifact_and_block() { + use crate::commands::test_support::SweepTestApp; + let dir = tempfile::tempdir().expect("checkpoint parent"); + let prefix = dir.path().join("checkpoint"); + let mut app = SweepTestApp; + dump_info::create_dump_dir_with_info( + &mut app, + &prefix, + &dump_info::DumpInfo::at_baseline(3), + ) + .expect("create artifact"); + assert!(matches!( + Checkpoint::::load(&prefix, 10), + Err(SetupRecoveryError::CheckpointLoad { .. }) + )); + let write_receipt = |nonce| { + let receipt = dump_info::CheckpointInfo { + format_version: dump_info::FORMAT_VERSION, + next_batch_nonce: nonce, + inclusion_block: 10, + }; + std::fs::write( + prefix.join("checkpoint.toml"), + toml::to_string(&receipt).unwrap(), + ) + .expect("write receipt"); + }; + write_receipt(4); + assert!(matches!( + Checkpoint::::load(&prefix, 10), + Err(SetupRecoveryError::CheckpointLoad { .. }) + )); + write_receipt(3); + assert!(matches!( + Checkpoint::::load(&prefix, 11), + Err(SetupRecoveryError::CheckpointLoad { .. }) + )); + let checkpoint = Checkpoint::::load(&prefix, 10).unwrap(); + assert_eq!( + (checkpoint.checkpoint_nonce, checkpoint.checkpoint_block), + (3, 10) + ); + } + + #[test] + fn genesis_checkpoint_loads_and_has_an_empty_seed_interval() { + use crate::commands::test_support::SweepTestApp; + let dir = tempfile::tempdir().expect("checkpoint parent"); + let prefix = dir.path().join("genesis"); + let mut app = SweepTestApp; + dump_info::create_dump_dir_with_info( + &mut app, + &prefix, + &dump_info::DumpInfo::at_baseline(0), + ) + .expect("genesis artifact"); + std::fs::write( + prefix.join("checkpoint.toml"), + toml::to_string(&dump_info::CheckpointInfo { + format_version: dump_info::FORMAT_VERSION, + next_batch_nonce: 0, + inclusion_block: 0, + }) + .unwrap(), + ) + .expect("genesis receipt"); + let checkpoint = Checkpoint::::load(&prefix, 0).expect("load genesis export"); + assert_eq!(checkpoint.executed_safe_block, checkpoint.checkpoint_block); + let db = temp_db("genesis-checkpoint-seed"); + let mut storage = Storage::open(&db.path).expect("storage"); + storage + .append_safe_inputs( + 10, + &[StoredSafeInput { + sender: Address::repeat_byte(0x22), + payload: vec![1], + block_number: 5, + }], + SENDER_A, + &default_protocol_timing(), + ) + .expect("post-genesis direct"); + let (seeds, replay) = source_fold_inputs(&mut storage, &checkpoint, 10, SENDER_A) + .expect("source empty seed and replay"); + assert!(seeds.is_empty()); + assert_eq!(replay.len(), 1); + assert_eq!(replay[0].inclusion_block, 5); + } + + #[test] + fn non_genesis_checkpoint_rejects_equal_execution_and_inclusion_blocks() { + use app_core::application::{WalletApp, WalletConfig}; + use sequencer_core::application::execute_direct_input; + use sequencer_core::l2_tx::DirectInput; + + let dir = tempfile::tempdir().expect("checkpoint parent"); + let prefix = dir.path().join("checkpoint"); + let mut app = WalletApp::new(WalletConfig::devnet()); + execute_direct_input( + &mut app, + &DirectInput { + sender: Address::repeat_byte(0x22), + payload: vec![], + block_number: 10, + }, + ) + .expect("execute direct at checkpoint block"); + dump_info::create_dump_dir_with_info( + &mut app, + &prefix, + &dump_info::DumpInfo::at_batch_close(0), + ) + .expect("checkpoint artifact"); + std::fs::write( + prefix.join("checkpoint.toml"), + toml::to_string(&dump_info::CheckpointInfo { + format_version: dump_info::FORMAT_VERSION, + next_batch_nonce: 1, + inclusion_block: 10, + }) + .unwrap(), + ) + .expect("checkpoint receipt"); + + // A direct arriving after this batch in the same L1 block is still + // pending, but the (A, B] seed interval would omit the whole block. + let error = Checkpoint::::load(&prefix, 10).err().unwrap(); + assert!(matches!( + error, + SetupRecoveryError::CheckpointNotBeforeBlock { + executed_safe_block: 10, + checkpoint_block: 10, + } + )); + assert!(error.to_string().contains("pending same-block directs")); + } + #[test] fn completed_plain_setup_is_a_noop_and_writes_nothing() { let db = temp_db("setup-noop-preserves-recovery"); @@ -764,9 +907,14 @@ mod tests { Storage::initialize_for_command(db.path.as_str(), storage::LifecycleCommand::Setup) .expect("initialize"); storage - .insert_initial_finalized_dump(&db._dir.path().join("finalized"), 0, 0, 0, 0) - .expect("register finalized snapshot"); - storage.complete_setup().expect("complete setup"); + .complete_baseline_setup( + &db._dir.path().join("baseline"), + sequencer_core::history::ExecutedInputCount::ZERO, + 0, + 0, + false, + ) + .expect("complete setup"); storage .record_terminal_fault(storage::LifecycleCommand::Run, "prior terminal death") .expect("record prior fault"); diff --git a/sequencer/src/commands/test_support.rs b/sequencer/src/commands/test_support.rs index 5ddc0d4b..f71d5afe 100644 --- a/sequencer/src/commands/test_support.rs +++ b/sequencer/src/commands/test_support.rs @@ -83,8 +83,6 @@ pub(crate) fn create_structured_dump(dump_dir: &std::path::Path) { &dump_info::DumpInfo { format_version: dump_info::FORMAT_VERSION, next_batch_nonce: 0, - l2_tx_index: 0, - promoted_inclusion_block: None, }, ) .expect("create structured dump"); diff --git a/sequencer/src/egress/api/snapshot.rs b/sequencer/src/egress/api/snapshot.rs index b7dc4a8c..1c83041a 100644 --- a/sequencer/src/egress/api/snapshot.rs +++ b/sequencer/src/egress/api/snapshot.rs @@ -1,31 +1,16 @@ // (c) Cartesi and individual authors (see AUTHORS) // SPDX-License-Identifier: Apache-2.0 (see LICENSE) -//! Operator-only snapshot read endpoints. -//! -//! **These routes are operator-internal.** `/finalized_state` and -//! `/latest_snapshot` stream full application state with no authentication -//! and MUST NOT be exposed to the public internet — they serve the watchdog -//! and the operator's indexers from the internal tier, gated by network -//! controls today (and bound to the internal listener once the per-port api -//! split lands). See `AGENTS.md` and the threat model. -//! -//! - `GET /finalized_state/inclusion_block` — cheap JSON -//! `{ inclusion_block, l2_tx_index }` the watchdog polls to detect advance. -//! - `GET /finalized_state` — streams the finalized state file (watchdog). -//! - `GET /latest_snapshot` — streams the latest snapshot dump (indexers). -//! -//! The two streaming routes lease the dump for the lifetime of the response — -//! acquired atomically with the row read, so GC can't delete it between the -//! read and the file open — and release it via a drop-guard that fires even -//! on client disconnect. `Storage::reset_dump_leases` at startup is the crash -//! backstop. +//! Operator-only comparison files and complete restore/recovery archives. +//! Snapshot selection, history metadata, and leases are one SQLite transaction. +use std::future::Future; use std::path::{Path, PathBuf}; use std::pin::Pin; use std::sync::Arc; use std::task::{Context, Poll}; +use crate::ingress::inclusion_lane::dump_info::{self, CheckpointInfo}; use axum::Json; use axum::Router; use axum::body::Body; @@ -36,7 +21,7 @@ use axum::routing::get; use serde::Serialize; use tokio::fs::File; use tokio::io::{AsyncRead, ReadBuf}; -use tokio_util::io::ReaderStream; +use tokio_util::io::{ReaderStream, SyncIoBridge}; use crate::http::{StorageTaskError, storage_task}; use crate::runtime::shutdown::{RuntimeScope, abort_terminal}; @@ -77,13 +62,14 @@ pub(crate) fn router( get(finalized_inclusion_block), ) .route("/latest_snapshot", get(latest_snapshot)) + .route("/finalized_snapshot", get(finalized_snapshot)) .with_state(state) } #[derive(Serialize)] struct InclusionBlockResponse { inclusion_block: u64, - l2_tx_index: u64, + executed_input_count: u64, } /// `GET /finalized_state/inclusion_block` — cheap read, no lease (no file is @@ -99,7 +85,7 @@ async fn finalized_inclusion_block(State(state): State>) - match result { Ok(Some(finalized)) => Json(InclusionBlockResponse { inclusion_block: finalized.inclusion_block, - l2_tx_index: finalized.l2_tx_index, + executed_input_count: finalized.executed_input_count.get(), }) .into_response(), Ok(None) => StatusCode::NOT_FOUND.into_response(), @@ -129,7 +115,8 @@ async fn finalized_state( } let path = state_file_path(&state.snapshot, &leased.prefix); - let l2_tx_index = leased.l2_tx_index; + let executed_input_count = leased.executed_input_count.get(); + let history = leased.history_version; let LeasedDump { guard, .. } = leased; match File::open(&path).await { @@ -138,7 +125,12 @@ async fn finalized_state( .header(header::CONTENT_TYPE, "application/octet-stream") .header(header::ETAG, etag) .header("X-Inclusion-Block", inclusion_block.to_string()) - .header("X-L2-Tx-Index", l2_tx_index.to_string()) + .header("X-Executed-Input-Count", executed_input_count.to_string()) + .header("X-History-Era", history.era_id.to_string()) + .header( + "X-Recovery-Generation", + history.recovery_generation.get().to_string(), + ) .body(stream_body(file, guard)) .expect("snapshot response headers are well-formed"), // `guard` is a local here; on this error path it drops → lease released. @@ -153,35 +145,100 @@ async fn finalized_state( } } -/// `GET /latest_snapshot` — stream the latest snapshot dump (indexers: fetch -/// then subscribe at this offset). Latest pending if any, else finalized. +/// Full app-owned restore artifact, which may still be optimistic. async fn latest_snapshot(State(state): State>) -> Response { - let leased = match acquire_latest(&state).await { - Ok(Some(leased)) => leased, - Ok(None) => return StatusCode::NOT_FOUND.into_response(), - Err(err) => return internal_error("acquire latest snapshot lease", err), - }; + match acquire_latest(&state).await { + Ok(Some(leased)) => archive_response(&state, leased, None), + Ok(None) => StatusCode::NOT_FOUND.into_response(), + Err(err) => internal_error("acquire latest snapshot lease", err), + } +} - let path = state_file_path(&state.snapshot, &leased.prefix); - let l2_tx_index = leased.l2_tx_index; - let LeasedDump { guard, .. } = leased; +/// A self-contained operator backup: immutable restore artifact plus a receipt +/// identifying the accepted canonical boundary, selected under the same lease. +async fn finalized_snapshot(State(state): State>) -> Response { + match acquire_finalized(&state).await { + Ok(Some(FinalizedLease { + inclusion_block, + dump, + })) => { + let checkpoint = CheckpointInfo { + format_version: dump_info::FORMAT_VERSION, + next_batch_nonce: dump.next_batch_nonce, + inclusion_block, + }; + archive_response(&state, dump, Some(checkpoint)) + } + Ok(None) => StatusCode::NOT_FOUND.into_response(), + Err(err) => internal_error("acquire accepted snapshot lease", err), + } +} - match File::open(&path).await { - Ok(file) => Response::builder() - .status(StatusCode::OK) - .header(header::CONTENT_TYPE, "application/octet-stream") - .header("X-L2-Tx-Index", l2_tx_index.to_string()) - .body(stream_body(file, guard)) - .expect("snapshot response headers are well-formed"), - Err(err) => { - if err.kind() == std::io::ErrorKind::NotFound { +fn archive_response( + state: &SnapshotApiState, + leased: LeasedDump, + checkpoint: Option, +) -> Response { + let LeasedDump { + prefix, + executed_input_count, + history_version, + next_batch_nonce, + guard, + } = leased; + let guard = Arc::new(guard); + let producer_guard = guard.clone(); + let scope = state.shutdown.clone(); + let inclusion_block = checkpoint.as_ref().map(|c| c.inclusion_block); + let (writer, reader) = tokio::io::duplex(64 * 1024); + let (done_tx, done_rx) = tokio::sync::oneshot::channel(); + tokio::task::spawn_blocking(move || { + let _runtime_lifetime = scope; + let _lease = producer_guard; + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + dump_info::write_archive( + SyncIoBridge::new(writer), + &prefix, + next_batch_nonce, + checkpoint.as_ref(), + ) + })); + let result = match result { + Ok(Err(err)) if dump_info::referenced_artifact_io_is_terminal(&err) => { abort_terminal(format!( - "durable latest snapshot artifact missing: {path:?}" - )); + "durable snapshot archive is unusable: {}: {err}", + prefix.display() + )) } - internal_error("open latest snapshot file", err) - } + Ok(result) => result, + Err(_) => abort_terminal("snapshot archive producer panicked"), + }; + let _ = done_tx.send(result); + }); + let mut response = Response::builder() + .status(StatusCode::OK) + .header(header::CONTENT_TYPE, "application/x-tar") + .header("X-History-Era", history_version.era_id.to_string()) + .header( + "X-Recovery-Generation", + history_version.recovery_generation.get().to_string(), + ) + .header( + "X-Executed-Input-Count", + executed_input_count.get().to_string(), + ); + if let Some(block) = inclusion_block { + response = response.header("X-Inclusion-Block", block.to_string()); } + response + .body(Body::from_stream(ReaderStream::new(GuardedReader { + file: ArchiveReader { + reader, + done: Some(done_rx), + }, + _guard: guard, + }))) + .expect("snapshot headers are well-formed") } fn state_file_path(state: &SnapshotState, prefix: &Path) -> PathBuf { @@ -193,10 +250,47 @@ fn state_file_path(state: &SnapshotState, prefix: &Path) -> PathBuf { fn stream_body(file: File, guard: LeaseGuard) -> Body { Body::from_stream(ReaderStream::new(GuardedReader { file, - _guard: guard, + _guard: Arc::new(guard), })) } +/// Do not turn a producer failure into a successful truncated archive response. +struct ArchiveReader { + reader: tokio::io::DuplexStream, + done: Option>>, +} + +impl AsyncRead for ArchiveReader { + fn poll_read( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &mut ReadBuf<'_>, + ) -> Poll> { + let this = self.get_mut(); + if buf.remaining() == 0 { + return Poll::Ready(Ok(())); + } + let before = buf.filled().len(); + match Pin::new(&mut this.reader).poll_read(cx, buf) { + Poll::Ready(Ok(())) if buf.filled().len() == before => { + let Some(done) = this.done.as_mut() else { + return Poll::Ready(Ok(())); + }; + match Pin::new(done).poll(cx) { + Poll::Ready(result) => { + this.done = None; + Poll::Ready(result.unwrap_or_else(|_| { + Err(std::io::Error::other("snapshot producer terminated")) + })) + } + Poll::Pending => Poll::Pending, + } + } + other => other, + } + } +} + // ── Lease acquisition (storage returns the dump bundled with its release) ── async fn acquire_finalized(state: &SnapshotApiState) -> Result, BoxError> { @@ -243,12 +337,12 @@ async fn acquire_latest(state: &SnapshotApiState) -> Result, /// A file reader that also owns the lease guard. When the response body is /// dropped — stream completion, I/O error, or client disconnect — the guard /// drops with it and releases the lease. -struct GuardedReader { - file: File, - _guard: LeaseGuard, +struct GuardedReader { + file: R, + _guard: Arc, } -impl AsyncRead for GuardedReader { +impl AsyncRead for GuardedReader { fn poll_read( self: Pin<&mut Self>, cx: &mut Context<'_>, @@ -279,16 +373,22 @@ mod tests { use super::*; use crate::storage::test_helpers::temp_db; + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[cfg(unix)] - async fn panicking_state_path_aborts(test_name: &str, finalized: bool) { - if !crate::runtime::shutdown::abort_test_child(test_name) { + async fn finalized_state_path_panic_aborts_process() { + if !crate::runtime::shutdown::abort_test_child( + "egress::api::snapshot::tests::finalized_state_path_panic_aborts_process", + ) { return; } let db = temp_db("panicking-snapshot-path"); let mut storage = Storage::open(&db.path).expect("open storage"); storage - .insert_finalized_dump(Path::new("/tmp/panicking-snapshot-path"), 12, 34) - .expect("insert finalized snapshot"); + .insert_baseline_snapshot( + Path::new("/tmp/panicking-snapshot-path"), + crate::storage::ExecutedInputCount::ZERO, + ) + .expect("insert baseline snapshot"); drop(storage); let state = Arc::new(SnapshotApiState { snapshot: SnapshotState { @@ -300,35 +400,52 @@ mod tests { }); // Exercise the request-task boundary that would swallow this panic. - let result = tokio::spawn(async move { - if finalized { - finalized_state(State(state), HeaderMap::new()).await - } else { - latest_snapshot(State(state)).await - } - }) - .await; + let result = + tokio::spawn(async move { finalized_state(State(state), HeaderMap::new()).await }) + .await; panic!("snapshot path callback panic did not abort the process: {result:?}"); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - #[cfg(unix)] - async fn finalized_state_path_panic_aborts_process() { - panicking_state_path_aborts( - "egress::api::snapshot::tests::finalized_state_path_panic_aborts_process", - true, - ) - .await; - } - - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - #[cfg(unix)] - async fn latest_snapshot_path_panic_aborts_process() { - panicking_state_path_aborts( - "egress::api::snapshot::tests::latest_snapshot_path_panic_aborts_process", - false, - ) - .await; + async fn latest_snapshot_archives_opaque_state_without_comparison_callback() { + let db = temp_db("archive-without-comparison-path"); + let root = tempfile::tempdir().expect("snapshot root"); + let dump_dir = root.path().join("dump"); + std::fs::create_dir(&dump_dir).expect("create dump directory"); + std::fs::write(dump_info::app_prefix(&dump_dir), b"opaque restore artifact") + .expect("write app-owned state"); + dump_info::write_info(&dump_dir, &dump_info::DumpInfo::at_baseline(0)) + .expect("write immutable dump metadata"); + let mut storage = Storage::open(&db.path).expect("open storage"); + storage + .insert_baseline_snapshot(&dump_dir, crate::storage::ExecutedInputCount::ZERO) + .expect("insert baseline snapshot"); + drop(storage); + let state = Arc::new(SnapshotApiState { + snapshot: SnapshotState { + db_path: db.path, + state_file_in_dump: |_| panic!("archive must not ask for comparison bytes"), + }, + shutdown: RuntimeScope::default(), + release_scheduler: Arc::new(|release| release()), + }); + let response = latest_snapshot(State(state)).await; + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + response.headers()[header::CONTENT_TYPE], + "application/x-tar" + ); + let body = axum::body::to_bytes(response.into_body(), 64 * 1024) + .await + .expect("read complete archive"); + let restored = root.path().join("restored"); + tar::Archive::new(body.as_ref()) + .unpack(&restored) + .expect("unpack snapshot"); + assert_eq!( + std::fs::read(dump_info::app_prefix(&restored)).unwrap(), + b"opaque restore artifact", + ); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] @@ -342,15 +459,20 @@ mod tests { let db = temp_db("corrupt-finalized-endpoint"); let mut storage = Storage::open(db.path.as_str()).expect("open storage"); storage - .insert_finalized_dump(Path::new("/tmp/corrupt-finalized"), 12, 34) + .insert_baseline_snapshot( + Path::new("/tmp/corrupt-finalized"), + crate::storage::ExecutedInputCount::ZERO, + ) .expect("insert finalized snapshot"); drop(storage); let conn = Storage::open_connection(db.path.as_str()).expect("raw connection"); conn.pragma_update(None, "ignore_check_constraints", "ON") .expect("allow corruption fixture"); + conn.execute_batch("DROP TRIGGER trg_snapshot_immutable") + .unwrap(); conn.execute( - "UPDATE finalized_snapshot SET l2_tx_index = -1 WHERE singleton_id = 0", + "UPDATE snapshots SET executed_input_count = -1 WHERE batch_index IS NULL", [], ) .expect("corrupt finalized cursor"); @@ -379,7 +501,10 @@ mod tests { let db = temp_db("snapshot-gate-predicate"); let mut storage = Storage::open(db.path.as_str()).expect("open storage"); storage - .insert_finalized_dump(Path::new("/tmp/gate-finalized"), 12, 34) + .insert_baseline_snapshot( + Path::new("/tmp/gate-finalized"), + crate::storage::ExecutedInputCount::ZERO, + ) .expect("insert finalized snapshot"); drop(storage); @@ -411,7 +536,10 @@ mod tests { let db = temp_db("dangling-finalized-endpoint"); let mut storage = Storage::open(db.path.as_str()).expect("open storage"); storage - .insert_finalized_dump(Path::new("/tmp/dangling-finalized"), 12, 34) + .insert_baseline_snapshot( + Path::new("/tmp/dangling-finalized"), + crate::storage::ExecutedInputCount::ZERO, + ) .expect("insert finalized snapshot"); drop(storage); @@ -463,7 +591,10 @@ mod tests { let db = temp_db("missing-lease-row"); let mut storage = Storage::open(&db.path).expect("open storage"); storage - .insert_finalized_dump(Path::new("/tmp/lease-probe"), 12, 34) + .insert_baseline_snapshot( + Path::new("/tmp/lease-probe"), + crate::storage::ExecutedInputCount::ZERO, + ) .expect("register finalized dump"); drop(storage); let state = SnapshotApiState { @@ -496,12 +627,19 @@ mod tests { ) { return; } - let shutdown = RuntimeScope::default(); + let state = SnapshotApiState { + snapshot: SnapshotState { + db_path: String::new(), + state_file_in_dump: |prefix| prefix.join("state"), + }, + shutdown: RuntimeScope::default(), + release_scheduler: Arc::new(|release| release()), + }; let (started_tx, started_rx) = tokio::sync::oneshot::channel(); let (release_tx, release_rx) = std::sync::mpsc::channel(); let request = tokio::spawn(async move { storage_task::<(), _>( - shutdown, + state.shutdown.clone(), "cancelled request corruption probe", move |_scope| { started_tx.send(()).expect("started storage task"); diff --git a/sequencer/src/egress/api/state.rs b/sequencer/src/egress/api/state.rs index 078cd6fa..44a1487f 100644 --- a/sequencer/src/egress/api/state.rs +++ b/sequencer/src/egress/api/state.rs @@ -15,7 +15,6 @@ use crate::runtime::shutdown::RuntimeScope; pub(crate) struct SubscribeState { pub shutdown: RuntimeScope, pub ws_subscriber_limit: Arc, - pub ws_max_catchup_events: u64, pub tx_feed: L2TxFeed, } @@ -24,12 +23,10 @@ impl SubscribeState { shutdown: RuntimeScope, tx_feed: L2TxFeed, ws_max_subscribers: usize, - ws_max_catchup_events: u64, ) -> Self { Self { shutdown, ws_subscriber_limit: Arc::new(Semaphore::new(ws_max_subscribers)), - ws_max_catchup_events, tx_feed, } } diff --git a/sequencer/src/egress/api/subscribe.rs b/sequencer/src/egress/api/subscribe.rs index 53a544d0..9fc9d5d0 100644 --- a/sequencer/src/egress/api/subscribe.rs +++ b/sequencer/src/egress/api/subscribe.rs @@ -7,15 +7,20 @@ use std::sync::Arc; -use axum::extract::ws::{CloseFrame, Message, WebSocket, WebSocketUpgrade, close_code}; +use axum::extract::ws::{Message, WebSocket, WebSocketUpgrade}; use axum::extract::{Query, State}; use axum::response::{IntoResponse, Response}; use serde::Deserialize; use tokio::sync::OwnedSemaphorePermit; use tracing::warn; +use crate::egress::l2_tx_feed::Subscription; use crate::egress::l2_tx_feed::{BroadcastTxMessage, L2TxFeed, SubscribeError}; -use crate::http::WS_CATCHUP_WINDOW_EXCEEDED_REASON; +use crate::http::ApiError; +use axum::{Json, http::StatusCode}; +use sequencer_core::history::{ + EraId, ExecutedInputCount, HistoryClaim, HistoryVersion, RecoveryGeneration, +}; use super::SubscribeState; @@ -24,7 +29,9 @@ const MAX_INBOUND_WS_FRAME_SIZE: usize = 8 * 1024; #[derive(Debug, Deserialize)] pub(crate) struct SubscribeQuery { - from_offset: Option, + era_id: EraId, + recovery_generation: RecoveryGeneration, + next_input: ExecutedInputCount, } pub(crate) async fn subscribe_l2_txs( @@ -36,68 +43,50 @@ pub(crate) async fn subscribe_l2_txs( return err.into_response(); } - let from_offset = query.from_offset.unwrap_or(0); + let claim = HistoryClaim { + version: HistoryVersion { + era_id: query.era_id, + recovery_generation: query.recovery_generation, + }, + next_input: query.next_input, + }; let permit = match state.try_acquire_ws_subscriber_permit() { Ok(permit) => permit, Err(err) => return err.into_response(), }; let tx_feed = state.tx_feed.clone(); - let ws_max_catchup_events = state.ws_max_catchup_events; + let subscription = match tx_feed.subscribe_from(claim).await { + Ok(subscription) => subscription, + Err(SubscribeError::History(error)) => { + // WebSocket clients may stop reading after the HTTP headers. Keep + // the structured refusal available even when its body arrives later. + let policy = serde_json::to_string(&error).expect("history policy serializes"); + return ( + StatusCode::CONFLICT, + [("X-History-Error", policy)], + Json(error), + ) + .into_response(); + } + Err(error) => { + warn!(%error, "ws subscription unavailable"); + return ApiError::unavailable("subscription unavailable").into_response(); + } + }; ws.max_message_size(MAX_INBOUND_WS_MESSAGE_SIZE) .max_frame_size(MAX_INBOUND_WS_FRAME_SIZE) - .on_upgrade(move |socket| { - run_ws_session(tx_feed, socket, from_offset, permit, ws_max_catchup_events) - }) + .on_upgrade(move |socket| run_ws_session(tx_feed, socket, subscription, permit)) .into_response() } async fn run_ws_session( tx_feed: L2TxFeed, mut socket: WebSocket, - from_offset: u64, + mut subscription: Subscription, _subscriber_permit: OwnedSemaphorePermit, - ws_max_catchup_events: u64, ) { let shutdown = tx_feed.runtime_scope(); - let mut subscription = match tx_feed - .subscribe_from(from_offset, ws_max_catchup_events) - .await - { - Ok(subscription) => subscription, - Err(SubscribeError::CatchUpWindowExceeded { - requested_offset, - live_start_offset, - max_catchup_events, - }) => { - warn!( - requested_offset, - live_start_offset, - max_catchup_events, - "ws catch-up window exceeded; closing subscriber" - ); - let reason = format!( - "{WS_CATCHUP_WINDOW_EXCEEDED_REASON}: live_start_offset={live_start_offset}" - ); - close_with_frame(&mut socket, close_code::POLICY, reason.as_str()).await; - return; - } - Err(SubscribeError::OpenStorage { source }) => { - warn!(error = %source, "ws subscription failed to open replay storage"); - close_with_frame(&mut socket, close_code::ERROR, "subscription unavailable").await; - return; - } - Err(SubscribeError::LoadHeadOffset { source }) => { - warn!(error = %source, "ws subscription failed to read replay head"); - close_with_frame(&mut socket, close_code::ERROR, "subscription unavailable").await; - return; - } - Err(SubscribeError::Join { source }) => { - warn!(error = %source, "ws subscription preparation was cancelled"); - close_with_frame(&mut socket, close_code::ERROR, "subscription unavailable").await; - return; - } - }; loop { tokio::select! { @@ -134,17 +123,6 @@ async fn run_ws_session( } } -async fn close_with_frame(socket: &mut WebSocket, code: u16, reason: &str) { - let _ = send_ws_message( - socket, - Message::Close(Some(CloseFrame { - code, - reason: reason.into(), - })), - ) - .await; -} - async fn send_ws_event(socket: &mut WebSocket, event: &BroadcastTxMessage) -> Result<(), ()> { let payload = match serde_json::to_string(event) { Ok(value) => value, diff --git a/sequencer/src/egress/l2_tx_feed/error.rs b/sequencer/src/egress/l2_tx_feed/error.rs index 24f217ef..97234739 100644 --- a/sequencer/src/egress/l2_tx_feed/error.rs +++ b/sequencer/src/egress/l2_tx_feed/error.rs @@ -1,6 +1,7 @@ // (c) Cartesi and individual authors (see AUTHORS) // SPDX-License-Identifier: Apache-2.0 (see LICENSE) +use sequencer_core::history::HistoryPolicyError; use thiserror::Error; use crate::storage::{ @@ -24,14 +25,17 @@ pub enum SubscribeError { #[source] source: tokio::task::JoinError, }, - #[error( - "catch-up window exceeded: requested offset {requested_offset}, live start {live_start_offset}, max {max_catchup_events}" - )] - CatchUpWindowExceeded { - requested_offset: u64, - live_start_offset: u64, - max_catchup_events: u64, - }, + #[error(transparent)] + History(#[from] HistoryPolicyError), +} + +impl From for SubscribeError { + fn from(error: crate::storage::HistoryReadError) -> Self { + match error { + crate::storage::HistoryReadError::Policy(error) => Self::History(error), + crate::storage::HistoryReadError::Storage(source) => Self::LoadHeadOffset { source }, + } + } } impl SubscribeError { @@ -40,13 +44,15 @@ impl SubscribeError { Self::OpenStorage { source } => open_error_is_persistent(source), Self::LoadHeadOffset { source } => is_persistent_storage_error(source), Self::Join { source } => source.is_panic(), - Self::CatchUpWindowExceeded { .. } => false, + Self::History(_) => false, } } } #[derive(Debug, Error)] pub enum SubscriptionError { + #[error(transparent)] + History(HistoryPolicyError), #[error("cannot open subscription storage")] OpenStorage { #[source] @@ -68,6 +74,7 @@ pub enum SubscriptionError { impl SubscriptionError { pub(super) fn is_persistent_storage_invariant(&self) -> bool { match self { + Self::History(_) => false, Self::OpenStorage { source } => open_error_is_persistent(source), Self::LoadReplay { source, .. } => is_persistent_storage_error(source), Self::Join { source } => source.is_panic(), diff --git a/sequencer/src/egress/l2_tx_feed/mod.rs b/sequencer/src/egress/l2_tx_feed/mod.rs index 12ede2c9..376b9975 100644 --- a/sequencer/src/egress/l2_tx_feed/mod.rs +++ b/sequencer/src/egress/l2_tx_feed/mod.rs @@ -14,7 +14,7 @@ pub use sequencer_core::broadcast::BroadcastTxMessage; use std::panic::{AssertUnwindSafe, catch_unwind}; use std::time::Duration; -use alloy_primitives::Address; +use sequencer_core::history::{HistoryBounds, HistoryClaim}; use tokio::sync::mpsc; use crate::runtime::process_lock::spawn_blocking_with_lock; @@ -34,10 +34,6 @@ fn panic_message(payload: &dyn std::any::Any) -> &str { pub struct L2TxFeedConfig { pub idle_poll_interval: Duration, pub page_size: usize, - /// Address of the batch submitter wallet. Direct inputs from this sender - /// are skipped before WS delivery (they're our own batch submissions). - /// One of I11's three consumer-side sender checks — keep them in sync. - pub batch_submitter_address: Address, } #[derive(Clone)] @@ -45,7 +41,6 @@ pub struct L2TxFeed { db_path: String, page_size: usize, idle_poll_interval: Duration, - batch_submitter_address: Address, shutdown: RuntimeScope, } @@ -63,14 +58,11 @@ const DEFAULT_IDLE_POLL_INTERVAL: Duration = Duration::from_millis(20); const DEFAULT_PAGE_SIZE: usize = 256; const SUBSCRIPTION_BUFFER_CAPACITY: usize = 1024; -impl L2TxFeedConfig { - /// The only constructor: the submitter address is mandatory, so a feed - /// that fans out our own batch envelopes is unconstructible. - pub fn new(batch_submitter_address: Address) -> Self { +impl Default for L2TxFeedConfig { + fn default() -> Self { Self { idle_poll_interval: DEFAULT_IDLE_POLL_INTERVAL, page_size: DEFAULT_PAGE_SIZE, - batch_submitter_address, } } } @@ -81,64 +73,50 @@ impl L2TxFeed { db_path, page_size: config.page_size.max(1), idle_poll_interval: config.idle_poll_interval, - batch_submitter_address: config.batch_submitter_address, shutdown, } } - pub async fn subscribe_from( - &self, - from_offset: u64, - max_catchup_events: u64, - ) -> Result { - // Classify faults inside blocking work: a cancelled request cannot - // discard a persistent error or panic after its SQLite task starts. - // The task independently retains the process lock until it finishes. - let prepare = { - let db_path = self.db_path.clone(); - let batch_submitter_address = self.batch_submitter_address; - spawn_blocking_with_lock(self.shutdown.process_lock(), move || { - match catch_unwind(AssertUnwindSafe(|| { - load_catchup_info( - db_path.as_str(), - from_offset, - max_catchup_events, - batch_submitter_address, - ) - })) { - Ok(Err(error)) if error.is_persistent_storage_invariant() => { - abort_terminal(format_args!("preparing tx-feed subscription: {error}")); - } - Ok(result) => result, - Err(payload) => abort_terminal(format_args!( - "panic preparing tx-feed subscription: {}", - panic_message(&*payload) - )), + async fn prepare(&self, claim: HistoryClaim) -> Result { + let db_path = self.db_path.clone(); + let prepared = spawn_blocking_with_lock(self.shutdown.process_lock(), move || { + match catch_unwind(AssertUnwindSafe(|| { + let mut storage = Storage::open_read_only(&db_path) + .map_err(|source| SubscribeError::OpenStorage { source })?; + storage + .canonical_history_page(claim, 0) + .map(|page| page.bounds) + .map_err(SubscribeError::from) + })) { + Ok(Err(error)) if error.is_persistent_storage_invariant() => { + abort_terminal(format_args!("preparing tx-feed subscription: {error}")) } - }) - .await - }; - let (head_offset, catchup_events) = match prepare { - Ok(Ok(info)) => info, - Ok(Err(error)) => return Err(error), - Err(join) if join.is_panic() => { - abort_terminal(format_args!("preparing tx-feed subscription: {join}")) + Ok(result) => result, + Err(payload) => abort_terminal(format_args!( + "panic preparing tx-feed subscription: {}", + panic_message(&*payload) + )), } - Err(source) => return Err(SubscribeError::Join { source }), - }; - if catchup_events > max_catchup_events { - return Err(SubscribeError::CatchUpWindowExceeded { - requested_offset: from_offset, - live_start_offset: head_offset, - max_catchup_events, - }); + }) + .await; + match prepared { + Ok(result) => result, + Err(source) if source.is_panic() => { + abort_terminal(format_args!("preparing tx-feed subscription: {source}")) + } + Err(source) => Err(SubscribeError::Join { source }), } + } + pub async fn subscribe_from( + &self, + claim: HistoryClaim, + ) -> Result { + self.prepare(claim).await?; let (events_tx, events_rx) = mpsc::channel(SUBSCRIPTION_BUFFER_CAPACITY); let db_path = self.db_path.clone(); let page_size = self.page_size; let idle_poll_interval = self.idle_poll_interval; - let batch_submitter_address = self.batch_submitter_address; let shutdown = self.shutdown.clone(); let task = tokio::task::spawn_blocking(move || { match catch_unwind(AssertUnwindSafe(|| { @@ -146,8 +124,7 @@ impl L2TxFeed { db_path.as_str(), page_size, idle_poll_interval, - batch_submitter_address, - from_offset, + claim, shutdown.clone(), events_tx, ) @@ -200,68 +177,48 @@ impl Subscription { } } -/// Returns `(head_offset, broadcastable_event_count_after_from_offset)`. -/// -/// Counts events the client will actually receive — excludes invalidated batches -/// and batch-submitter direct inputs (which are filtered before WS delivery). -fn load_catchup_info( - db_path: &str, - from_offset: u64, - max_catchup_events: u64, - batch_submitter_address: Address, -) -> Result<(u64, u64), SubscribeError> { - let mut storage = Storage::open_read_only(db_path) - .map_err(|source| SubscribeError::OpenStorage { source })?; - let head_offset = storage - .ordered_l2_tx_head_offset() - .map_err(|source| SubscribeError::LoadHeadOffset { source })?; - let catchup_count = storage - .count_broadcastable_events_after( - from_offset, - max_catchup_events.saturating_add(1), - batch_submitter_address, - ) - .map_err(|source| SubscribeError::LoadHeadOffset { source })?; - Ok((head_offset, catchup_count)) -} - fn run_subscription( db_path: &str, page_size: usize, idle_poll_interval: Duration, - batch_submitter_address: Address, - from_offset: u64, + mut claim: HistoryClaim, shutdown: RuntimeScope, events_tx: mpsc::Sender, ) -> Result<(), SubscriptionError> { let mut storage = Storage::open_read_only(db_path) .map_err(|source| SubscriptionError::OpenStorage { source })?; - let mut next_offset = from_offset; loop { if shutdown.is_shutdown_requested() || events_tx.is_closed() { return Ok(()); } - let txs = storage - .ordered_l2_tx_rows_page_from(next_offset, page_size) - .map_err(|source| SubscriptionError::LoadReplay { - offset: next_offset, - source, - })?; - - if txs.is_empty() { + let page = + storage + .canonical_history_page(claim, page_size) + .map_err(|error| match error { + crate::storage::HistoryReadError::Storage(source) => { + SubscriptionError::LoadReplay { + offset: claim.next_input.get(), + source, + } + } + crate::storage::HistoryReadError::Policy(source) => { + SubscriptionError::History(source) + } + })?; + claim = page.next_claim(); + if page.rows.is_empty() { std::thread::sleep(idle_poll_interval); continue; } - for row in txs { + for row in page.rows { if shutdown.is_shutdown_requested() || events_tx.is_closed() { return Ok(()); } - next_offset = row.offset; - let offset = row.offset; + let offset = row.offset.get(); let event = match row.context { L2TxContext::UserOp { tx, @@ -277,19 +234,14 @@ fn run_subscription( block_timestamp, transaction_hash, .. - } => { - if tx.sender == batch_submitter_address { - continue; - } - BroadcastTxMessage::from_direct_input( - offset, - tx, - input_index, - batch_nonce, - block_timestamp, - transaction_hash, - ) - } + } => BroadcastTxMessage::from_direct_input( + offset, + tx, + input_index, + batch_nonce, + block_timestamp, + transaction_hash, + ), }; if events_tx.blocking_send(event).is_err() { return Ok(()); diff --git a/sequencer/src/egress/l2_tx_feed/tests.rs b/sequencer/src/egress/l2_tx_feed/tests.rs index 5b5e3c51..e23a1d94 100644 --- a/sequencer/src/egress/l2_tx_feed/tests.rs +++ b/sequencer/src/egress/l2_tx_feed/tests.rs @@ -7,11 +7,14 @@ use alloy_primitives::{Address, B256, Signature}; use tokio::sync::oneshot; use super::{BroadcastTxMessage, L2TxFeed, L2TxFeedConfig, SubscribeError}; -use crate::ingress::inclusion_lane::{PendingUserOp, SequencerError}; +use crate::ingress::inclusion_lane::{IncludedUserOp, PendingUserOp, SequencerError}; use crate::runtime::process_lock::{ProcessLock, ProcessLockError}; use crate::runtime::shutdown::RuntimeScope; -use crate::storage::test_helpers::temp_db; -use crate::storage::{FrontierMode, IngestedSafeInput, SafeInputRange, Storage, StoredSafeInput}; +use crate::storage::test_helpers::{pin_test_deployment_identity, temp_db}; +use crate::storage::{FrontierMode, IngestedSafeInput, SafeInputRange, Storage}; +use sequencer_core::history::{ + ExecutedInputCount, HistoryClaim, HistoryPolicyError, RecoveryGeneration, +}; use sequencer_core::l2_tx::{DirectInput, ValidUserOp}; use sequencer_core::user_op::UserOp; @@ -65,36 +68,71 @@ fn broadcast_direct_input_serializes_with_hex_payload() { } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn subscribe_from_rejects_catchup_window() { - let db = temp_db("catchup-window"); - seed_ordered_txs(db.path.as_str()); - append_direct_input(db.path.as_str()); - let feed = test_feed(db.path.as_str(), RuntimeScope::default()); - - let result = feed.subscribe_from(1, 1).await; - +async fn subscription_refuses_ahead_and_stale_claims() { + let db = temp_db("subscription-claims"); + seed_ordered_txs(&db.path); + let feed = test_feed(&db.path, RuntimeScope::default()); + assert!(matches!(feed.subscribe_from(claim(&db.path, 3)).await, + Err(SubscribeError::History(HistoryPolicyError::AheadOfHead { head })) if head.get() == 2)); + let mut stale = claim(&db.path, 0); + stale.version.recovery_generation = RecoveryGeneration::new(1); assert!(matches!( - result, - Err(SubscribeError::CatchUpWindowExceeded { - requested_offset: 1, - live_start_offset: 3, - max_catchup_events: 1, - }) + feed.subscribe_from(stale).await, + Err(SubscribeError::History( + HistoryPolicyError::StaleGeneration { .. } + )) )); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn subscribe_from_accepts_exact_catchup_window() { - let db = temp_db("catchup-window-exact"); - seed_ordered_txs(db.path.as_str()); - let feed = test_feed(db.path.as_str(), RuntimeScope::default()); - - let subscription = feed.subscribe_from(0, 2).await; - - assert!( - subscription.is_ok(), - "exactly 2 replayable events should be allowed" +async fn subscription_accepts_more_than_fifty_thousand_inputs_without_a_total_cap() { + let db = temp_db("subscription-deep-history"); + let mut storage = Storage::open(&db.path).unwrap(); + let mut head = storage + .initialize_open_state(0, SafeInputRange::empty_at(0)) + .unwrap(); + let inputs: Vec<_> = (0..50_001) + .map(|offset| { + let (respond_to, _) = oneshot::channel(); + IncludedUserOp { + pending: PendingUserOp { + signed: sequencer_core::user_op::SignedUserOp { + sender: Address::repeat_byte(0x11), + signature: Signature::test_signature(), + user_op: UserOp { + nonce: offset, + max_fee: u16::MAX, + data: vec![0x42].into(), + }, + }, + respond_to, + received_at: SystemTime::now(), + }, + executed_input_offset: ExecutedInputCount::new(u64::from(offset)), + } + }) + .collect(); + storage + .append_executed_user_ops_chunk(&mut head, &inputs) + .unwrap(); + drop(storage); + let feed = L2TxFeed::new( + db.path.clone(), + RuntimeScope::default(), + L2TxFeedConfig { + page_size: 2, + ..Default::default() + }, ); + let mut subscription = feed.subscribe_from(claim(&db.path, 0)).await.unwrap(); + for expected in 0..3 { + let event = tokio::time::timeout(Duration::from_secs(2), subscription.recv()) + .await + .unwrap() + .unwrap(); + assert_eq!(event.offset(), expected); + } + subscription.finish().await.unwrap(); } #[test] @@ -103,6 +141,7 @@ fn cancelled_catchup_prepare_retains_process_lock_until_blocking_read_finishes() seed_ordered_txs(db.path.as_str()); let data_dir = db._dir.path().to_str().expect("utf8 data dir").to_string(); let db_path = db.path.clone(); + let start = claim(&db_path, 0); let runtime = tokio::runtime::Builder::new_current_thread() .max_blocking_threads(1) .enable_all() @@ -126,7 +165,7 @@ fn cancelled_catchup_prepare_retains_process_lock_until_blocking_read_finishes() let (subscribe_entered_tx, subscribe_entered_rx) = oneshot::channel(); let subscribe = tokio::spawn(async move { let _ = subscribe_entered_tx.send(()); - feed.subscribe_from(0, u64::MAX).await + feed.subscribe_from(start).await }); subscribe_entered_rx .await @@ -170,7 +209,10 @@ async fn subscription_replays_existing_rows_in_order() { seed_ordered_txs(db.path.as_str()); let feed = test_feed(db.path.as_str(), RuntimeScope::default()); - let mut subscription = feed.subscribe_from(0, u64::MAX).await.expect("subscribe"); + let mut subscription = feed + .subscribe_from(claim(&db.path, 0)) + .await + .expect("subscribe"); let first = tokio::time::timeout(Duration::from_secs(1), subscription.recv()) .await @@ -184,7 +226,7 @@ async fn subscription_replays_existing_rows_in_order() { assert!(matches!( first, BroadcastTxMessage::UserOp { - offset: 1, + offset: 0, nonce: 7, safe_block: 123, batch_nonce: 1, @@ -194,7 +236,7 @@ async fn subscription_replays_existing_rows_in_order() { assert!(matches!( second, BroadcastTxMessage::DirectInput { - offset: 2, + offset: 1, input_index: 0, batch_nonce: 1, block_timestamp: 1_700_000_000, @@ -206,43 +248,6 @@ async fn subscription_replays_existing_rows_in_order() { subscription.finish().await.expect("finish subscription"); } -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn subscription_filters_batch_submitter_safe_inputs() { - let db = temp_db("filters-batch-submitter-inputs"); - let batch_submitter_address = Address::from([0xfe; 20]); - seed_ordered_txs_with_sender(db.path.as_str(), batch_submitter_address); - let feed = L2TxFeed::new( - db.path.clone(), - RuntimeScope::default(), - L2TxFeedConfig { - idle_poll_interval: Duration::from_millis(2), - page_size: 64, - ..L2TxFeedConfig::new(batch_submitter_address) - }, - ); - - let mut subscription = feed.subscribe_from(0, u64::MAX).await.expect("subscribe"); - let first = tokio::time::timeout(Duration::from_secs(1), subscription.recv()) - .await - .expect("wait first event") - .expect("first event"); - - // DB offsets start at 1. The user op is the first sequenced tx (offset=1), - // and the batch submitter's safe input (offset=2) is filtered out. - assert!(matches!( - first, - BroadcastTxMessage::UserOp { offset: 1, .. } - )); - - let no_second = tokio::time::timeout(Duration::from_millis(50), subscription.recv()).await; - assert!( - no_second.is_err(), - "filtered batch-submitter input should not be broadcast" - ); - - subscription.finish().await.expect("finish subscription"); -} - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn shutdown_signal_closes_subscription() { let db = temp_db("shutdown-closes"); @@ -251,7 +256,7 @@ async fn shutdown_signal_closes_subscription() { let feed = test_feed(db.path.as_str(), shutdown.clone()); let mut subscription = feed - .subscribe_from(u64::MAX, u64::MAX) + .subscribe_from(claim(&db.path, 2)) .await .expect("subscribe"); @@ -277,14 +282,14 @@ async fn corrupt_feed_head_trips_terminal_storage_fault() { let db = temp_db("corrupt-feed-head"); seed_ordered_txs(db.path.as_str()); let conn = Storage::open_connection(db.path.as_str()).expect("raw connection"); - conn.execute("UPDATE sequenced_l2_txs SET offset = -offset", []) - .expect("corrupt offsets"); + let start = claim(&db.path, 0); + conn.execute_batch("PRAGMA ignore_check_constraints = ON; DROP TRIGGER trg_history_generation_monotonic; UPDATE history_state SET recovery_generation = 'broken';").expect("corrupt generation"); drop(conn); let shutdown = RuntimeScope::default(); let feed = test_feed(db.path.as_str(), shutdown.clone()); - let _ = feed.subscribe_from(0, u64::MAX).await; + let _ = feed.subscribe_from(start).await; panic!("corrupt feed head returned instead of aborting"); } @@ -305,161 +310,15 @@ async fn corrupt_feed_page_trips_terminal_storage_fault() { let shutdown = RuntimeScope::default(); let feed = test_feed(db.path.as_str(), shutdown.clone()); - let mut subscription = feed.subscribe_from(0, u64::MAX).await.expect("subscribe"); + let mut subscription = feed + .subscribe_from(claim(&db.path, 0)) + .await + .expect("subscribe"); let _ = subscription.recv().await; panic!("corrupt feed page returned instead of aborting"); } -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn catchup_window_not_inflated_by_invalidated_batch_holes() { - // Regression test: after batch invalidation, offset holes in sequenced_l2_txs - // must not inflate the catch-up event count. The check should count actual - // valid events, not subtract rowids. - let db = temp_db("catchup-holes"); - let mut storage = Storage::open(db.path.as_str()).expect("open storage"); - - // Create two closed batches, each with one direct input. - let mut head = storage - .initialize_open_state(0, SafeInputRange::empty_at(0)) - .expect("initialize"); - storage - .append_safe_inputs( - 10, - &[StoredSafeInput { - sender: Address::ZERO, - payload: vec![0xaa], - block_number: 10, - }], - Address::ZERO, - &sequencer_core::protocol::ProtocolTiming { - max_wait_blocks: sequencer_core::MAX_WAIT_BLOCKS, - preemptive_margin_blocks: 75, - l1_read_stale_after_blocks: 900, - seconds_per_block: 12, - }, - ) - .expect("append direct 0"); - storage - .close_frame_only(&mut head, 10, SafeInputRange::new(0, 1)) - .expect("close frame"); - storage - .close_frame_and_batch(&mut head, 10) - .expect("close batch 0"); - - storage - .append_safe_inputs( - 20, - &[StoredSafeInput { - sender: Address::ZERO, - payload: vec![0xbb], - block_number: 20, - }], - Address::ZERO, - &sequencer_core::protocol::ProtocolTiming { - max_wait_blocks: sequencer_core::MAX_WAIT_BLOCKS, - preemptive_margin_blocks: 75, - l1_read_stale_after_blocks: 900, - seconds_per_block: 12, - }, - ) - .expect("append direct 1"); - storage - .close_frame_only(&mut head, 20, SafeInputRange::new(1, 2)) - .expect("close frame"); - drop(storage); - - // Before invalidation: 2 valid events. - // With max_catchup_events=1, subscribing from 0 should fail. - let feed = test_feed(db.path.as_str(), RuntimeScope::default()); - assert!( - feed.subscribe_from(0, 1).await.is_err(), - "should reject: 2 valid events > max 1" - ); - - // Invalidate batch 0 — this creates a hole in the offset space. - // Now only 1 valid event remains (from batch 1). - let mut storage = Storage::open(db.path.as_str()).expect("reopen storage"); - storage.insert_invalid_batch(0).expect("invalidate batch 0"); - drop(storage); - - // After invalidation: only 1 valid event, so max_catchup_events=1 should succeed. - let feed = test_feed(db.path.as_str(), RuntimeScope::default()); - assert!( - feed.subscribe_from(0, 1).await.is_ok(), - "should accept: only 1 valid event after invalidation, despite rowid hole" - ); -} - -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn catchup_window_excludes_batch_submitter_direct_inputs() { - // Regression test: batch-submitter direct inputs are filtered before WS - // delivery, so the catch-up window must not count them. Otherwise a - // reconnecting client could be rejected even when the number of - // replayable messages is within the limit. - let db = temp_db("catchup-submitter-filter"); - let batch_submitter = Address::from([0xfe; 20]); - let user_address = Address::from([0x01; 20]); - - let mut storage = Storage::open(db.path.as_str()).expect("open storage"); - let mut head = storage - .initialize_open_state(0, SafeInputRange::empty_at(0)) - .expect("initialize"); - - // Two direct inputs: one from the batch submitter, one from a user. - storage - .append_safe_inputs( - 10, - &[ - StoredSafeInput { - sender: batch_submitter, - payload: vec![0xaa], - block_number: 10, - }, - StoredSafeInput { - sender: user_address, - payload: vec![0xbb], - block_number: 10, - }, - ], - Address::ZERO, - &sequencer_core::protocol::ProtocolTiming { - max_wait_blocks: sequencer_core::MAX_WAIT_BLOCKS, - preemptive_margin_blocks: 75, - l1_read_stale_after_blocks: 900, - seconds_per_block: 12, - }, - ) - .expect("append directs"); - storage - .close_frame_only(&mut head, 10, SafeInputRange::new(0, 2)) - .expect("close frame"); - drop(storage); - - // With a submitter address that matches no seeded sender: 2 events, - // max=1 should reject. - let feed_no_filter = L2TxFeed::new( - db.path.clone(), - RuntimeScope::default(), - L2TxFeedConfig::new(NO_OWN_BATCHES), - ); - assert!( - feed_no_filter.subscribe_from(0, 1).await.is_err(), - "without filter: 2 events > max 1" - ); - - // With batch_submitter_address filtering: only the user's event counts. - let feed_filtered = L2TxFeed::new( - db.path.clone(), - RuntimeScope::default(), - L2TxFeedConfig::new(batch_submitter), - ); - assert!( - feed_filtered.subscribe_from(0, 1).await.is_ok(), - "with filter: only 1 broadcastable event, should accept" - ); -} - /// Sentinel submitter for fixtures that seed no own-batch rows. Must not /// collide with any seeded sender (`seed_ordered_txs` uses `Address::ZERO`). const NO_OWN_BATCHES: Address = Address::repeat_byte(0x7f); @@ -471,17 +330,13 @@ fn test_feed(db_path: &str, shutdown: RuntimeScope) -> L2TxFeed { L2TxFeedConfig { idle_poll_interval: Duration::from_millis(2), page_size: 64, - ..L2TxFeedConfig::new(NO_OWN_BATCHES) }, ) } fn seed_ordered_txs(db_path: &str) { - seed_ordered_txs_with_sender(db_path, Address::ZERO); -} - -fn seed_ordered_txs_with_sender(db_path: &str, direct_sender: Address) { let mut storage = Storage::open(db_path).expect("open storage"); + pin_test_deployment_identity(&mut storage, NO_OWN_BATCHES); let mut head = storage .initialize_open_state(123, SafeInputRange::empty_at(0)) .expect("initialize open state"); @@ -505,20 +360,26 @@ fn seed_ordered_txs_with_sender(db_path: &str, direct_sender: Address) { }; storage - .append_user_ops_chunk(&mut head, &[pending]) + .append_executed_user_ops_chunk( + &mut head, + &[IncludedUserOp { + pending, + executed_input_offset: ExecutedInputCount::ZERO, + }], + ) .expect("append user-op chunk"); storage .append_ingested_safe_inputs_with_timestamp( 456, 456, &[IngestedSafeInput { - sender: direct_sender, + sender: Address::ZERO, payload: vec![0xaa], block_number: 456, block_timestamp: 1_700_000_000, transaction_hash: B256::repeat_byte(0xcd), }], - Address::ZERO, + NO_OWN_BATCHES, &sequencer_core::protocol::ProtocolTiming { max_wait_blocks: sequencer_core::MAX_WAIT_BLOCKS, preemptive_margin_blocks: 75, @@ -533,34 +394,10 @@ fn seed_ordered_txs_with_sender(db_path: &str, direct_sender: Address) { .expect("close frame with one drained direct input"); } -fn append_direct_input(db_path: &str) { - let mut storage = Storage::open(db_path).expect("open storage"); - let mut head = storage - .open_state() - .expect("load open state") - .expect("open state exists"); - storage - .append_ingested_safe_inputs_with_timestamp( - 789, - 789, - &[IngestedSafeInput { - sender: Address::ZERO, - payload: vec![0xbb], - block_number: 789, - block_timestamp: 1_700_000_001, - transaction_hash: B256::repeat_byte(0xef), - }], - Address::ZERO, - &sequencer_core::protocol::ProtocolTiming { - max_wait_blocks: sequencer_core::MAX_WAIT_BLOCKS, - preemptive_margin_blocks: 75, - l1_read_stale_after_blocks: 900, - seconds_per_block: 12, - }, - FrontierMode::Populate, - ) - .expect("append second direct input"); - storage - .close_frame_only(&mut head, 789, SafeInputRange::new(1, 2)) - .expect("close frame with second direct input"); +fn claim(db_path: &str, next: u64) -> HistoryClaim { + let storage = Storage::open_read_only(db_path).unwrap(); + HistoryClaim { + version: storage.history_state().unwrap().version, + next_input: ExecutedInputCount::new(next), + } } diff --git a/sequencer/src/harness.rs b/sequencer/src/harness.rs index 5df0c956..0b6c6fd3 100644 --- a/sequencer/src/harness.rs +++ b/sequencer/src/harness.rs @@ -168,15 +168,14 @@ mod tests { let mut storage = Storage::initialize_for_command(&db_path, LifecycleCommand::Setup) .expect("initialize setup"); storage - .insert_initial_finalized_dump( - &std::path::Path::new(&data_dir).join("finalized"), - 0, - 0, + .complete_baseline_setup( + &std::path::Path::new(&data_dir).join("baseline"), + sequencer_core::history::ExecutedInputCount::ZERO, 0, 0, + false, ) - .expect("register finalized snapshot"); - storage.complete_setup().expect("complete setup"); + .expect("complete setup"); drop(storage); let constructions = Arc::new(AtomicUsize::new(0)); diff --git a/sequencer/src/http.rs b/sequencer/src/http.rs index 9e7f5755..44284db0 100644 --- a/sequencer/src/http.rs +++ b/sequencer/src/http.rs @@ -142,15 +142,8 @@ impl IntoResponse for ApiError { // replace this with per-side starts on different ports. const DEFAULT_WS_MAX_SUBSCRIBERS: usize = 64; -const DEFAULT_WS_MAX_CATCHUP_EVENTS: u64 = 50_000; const DEFAULT_MAX_BODY_BYTES: usize = TxRequest::MAX_JSON_BYTES_RECOMMENDED; -/// Stable prefix of the WS Close-frame reason when the subscriber's requested -/// `from_offset` is too old for the catch-up window to bridge. -/// -/// The full reason is `{WS_CATCHUP_WINDOW_EXCEEDED_REASON}: live_start_offset=`. -pub const WS_CATCHUP_WINDOW_EXCEEDED_REASON: &str = "catch-up window exceeded"; - pub type ApiServerTask = JoinHandle>; type SnapshotReleaseTask = Box; @@ -285,7 +278,6 @@ pub struct ApiConfig { pub max_user_op_data_bytes: usize, pub max_body_bytes: usize, pub ws_max_subscribers: usize, - pub ws_max_catchup_events: u64, } impl ApiConfig { @@ -297,7 +289,6 @@ impl ApiConfig { max_user_op_data_bytes, max_body_bytes: DEFAULT_MAX_BODY_BYTES, ws_max_subscribers: DEFAULT_WS_MAX_SUBSCRIBERS, - ws_max_catchup_events: DEFAULT_WS_MAX_CATCHUP_EVENTS, } } } @@ -330,7 +321,6 @@ pub(crate) fn start_on_listener( shutdown.clone(), tx_feed, config.ws_max_subscribers, - config.ws_max_catchup_events, )); let app: Router = crate::ingress::api::router(submit_state, fee_state) .merge(crate::egress::api::router( diff --git a/sequencer/src/ingress/inclusion_lane/catch_up.rs b/sequencer/src/ingress/inclusion_lane/catch_up.rs index 85721a52..cfd7ce8c 100644 --- a/sequencer/src/ingress/inclusion_lane/catch_up.rs +++ b/sequencer/src/ingress/inclusion_lane/catch_up.rs @@ -1,179 +1,95 @@ // (c) Cartesi and individual authors (see AUTHORS) // SPDX-License-Identifier: Apache-2.0 (see LICENSE) -//! Startup-only replay: walk the persisted ordered-L2-tx stream and feed it -//! to the application so its in-memory state matches the DB before the lane -//! starts taking new work. Runs once, before the hot loop. +//! Restore the application's committed history suffix before runtime admission. use std::path::PathBuf; -use alloy_primitives::Address; - -use crate::storage::Storage; +use crate::storage::{HistoryReadError, L2TxContext, Storage}; use sequencer_core::application::{Application, execute_direct_input, execute_valid_user_op}; -use sequencer_core::history::ExecutedInputCount; -use sequencer_core::l2_tx::SequencedL2Tx; +use sequencer_core::history::{ExecutedInputCount, HistoryClaim}; use super::error::CatchUpError; const DEFAULT_CATCH_UP_PAGE_SIZE: usize = 256; -/// The single checkpoint the lane resumes from: the latest pending -/// snapshot if any, else the finalized snapshot. Carries both the dump -/// directory (whose `state` subtree loads the Application via -/// `from_dump`) and the matching `l2_tx_index` (the replay cursor), so -/// the loaded state and the catch-up cursor are guaranteed to come from -/// the *same* checkpoint. #[derive(Debug, Clone)] pub(super) struct CatchUpSnapshot { pub(super) dump_dir: PathBuf, - pub(super) l2_tx_index: u64, pub(super) executed_input_count: ExecutedInputCount, } -/// Select the resume checkpoint. Prefers the latest pending snapshot -/// (closest to current — less replay; safe because danger-zone recovery -/// clears any cascade-doomed pending *before* the lane starts, so a -/// surviving pending is either gold or legitimately in-flight), falling -/// back to the finalized snapshot. -/// -/// Returns the checkpoint directly rather than an `Option`: the -/// always-load invariant — `setup` registers the genesis finalized snapshot, -/// and `run` checks it during startup recovery and task-free runtime -/// preparation before `InclusionLane::start` — guarantees at least the -/// genesis finalized snapshot exists by the time -/// the lane resumes. Absence is a violated invariant (runtime/setup bug), -/// surfaced fail-loud as [`CatchUpError::NoSnapshot`]. pub(super) fn catch_up_snapshot(storage: &mut Storage) -> Result { - let (dump, l2_tx_index, executed_input_count) = storage + let snapshot = storage .latest_snapshot() .map_err(|source| CatchUpError::LoadSnapshot { source })? .ok_or(CatchUpError::NoSnapshot)?; Ok(CatchUpSnapshot { - dump_dir: dump.prefix, - l2_tx_index, - executed_input_count, + dump_dir: snapshot.dump.prefix, + executed_input_count: snapshot.executed_input_count, }) } pub(super) fn catch_up_application( app: &mut impl Application, storage: &mut Storage, - batch_submitter_address: Address, - start_offset: u64, + start: ExecutedInputCount, ) -> Result<(), CatchUpError> { - catch_up_application_paged( - app, - storage, - batch_submitter_address, - start_offset, - DEFAULT_CATCH_UP_PAGE_SIZE, - ) + catch_up_application_paged(app, storage, start, DEFAULT_CATCH_UP_PAGE_SIZE) } pub(super) fn catch_up_application_paged( app: &mut impl Application, storage: &mut Storage, - batch_submitter_address: Address, - start_offset: u64, + start: ExecutedInputCount, page_size: usize, ) -> Result<(), CatchUpError> { - // `start_offset` is the resume checkpoint's `l2_tx_index` — the - // global replay head captured when that snapshot's batch closed. The - // Application was loaded from the *same* checkpoint (see - // `catch_up_snapshot`), so replaying `offset > start_offset` applies - // exactly the txs not yet reflected in the loaded state. - // `ordered_l2_txs_page_from` uses `offset > ?1`. - let mut next_offset: u64 = start_offset; - let page_size = page_size.max(1); - + if app.executed_input_count() != start { + return Err(CatchUpError::SnapshotExecutionCountMismatch { + application: app.executed_input_count().get(), + storage: start.get(), + }); + } + let bounds = storage + .history_bounds() + .map_err(|source| CatchUpError::LoadReplay { + offset: start.get(), + source, + })?; + let mut claim = HistoryClaim { + version: bounds.version, + next_input: start, + }; loop { - let replay = storage - .ordered_l2_txs_page_from(next_offset, page_size) - .map_err(|source| CatchUpError::LoadReplay { - offset: next_offset, - source, + let page = storage + .canonical_history_page(claim, page_size.max(1)) + .map_err(|error| match error { + HistoryReadError::Storage(source) => CatchUpError::LoadReplay { + offset: claim.next_input.get(), + source, + }, + HistoryReadError::Policy(source) => CatchUpError::History(source), })?; - - if replay.is_empty() { + if page.rows.is_empty() { return Ok(()); } - - for row in replay { - let db_offset = row.db_offset; - replay_sequenced_l2_tx( - app, - batch_submitter_address, - db_offset, - row.tx, - row.frame_safe_block, - row.executed_input_offset, - )?; - next_offset = db_offset; - } - } -} - -fn replay_sequenced_l2_tx( - app: &mut impl Application, - batch_submitter_address: Address, - db_offset: u64, - item: SequencedL2Tx, - frame_safe_block: u64, - executed_input_offset: Option, -) -> Result<(), CatchUpError> { - match item { - SequencedL2Tx::UserOp(value) => { - assert_replay_mapping( - db_offset, - "user op", - Some(app.executed_input_count()), - executed_input_offset, - )?; - // The persisted covering frame's safe_block mirrors what the - // lane passed live, so the replayed app's safe-block clock - // lands on the same value. - execute_valid_user_op(app, &value, frame_safe_block) - .map(|_| ()) - .map_err(|source| CatchUpError::ReplayUserOp { source }) - } - SequencedL2Tx::Direct(direct) => { - if direct.sender == batch_submitter_address { - assert_replay_mapping( - db_offset, - "batch-submitter input", - None, - executed_input_offset, - )?; - return Ok(()); + claim = page.next_claim(); + for row in page.rows { + assert_eq!( + app.executed_input_count(), + row.offset, + "application replay offset differs from stored history" + ); + match row.context { + L2TxContext::UserOp { tx, safe_block, .. } => { + execute_valid_user_op(app, &tx, safe_block) + .map_err(|source| CatchUpError::ReplayUserOp { source })?; + } + L2TxContext::DirectInput { tx, .. } => { + execute_direct_input(app, &tx) + .map_err(|source| CatchUpError::ReplayDirectInput { source })?; + } } - - assert_replay_mapping( - db_offset, - "direct input", - Some(app.executed_input_count()), - executed_input_offset, - )?; - execute_direct_input(app, &direct) - .map(|_| ()) - .map_err(|source| CatchUpError::ReplayDirectInput { source }) } } } - -fn assert_replay_mapping( - db_offset: u64, - kind: &'static str, - expected: Option, - stored: Option, -) -> Result<(), CatchUpError> { - if expected == stored { - return Ok(()); - } - Err(CatchUpError::ExecutionOffsetMismatch { - db_offset, - kind, - expected: expected.map(ExecutedInputCount::get), - stored: stored.map(ExecutedInputCount::get), - }) -} diff --git a/sequencer/src/ingress/inclusion_lane/config.rs b/sequencer/src/ingress/inclusion_lane/config.rs index 276654f0..505768fc 100644 --- a/sequencer/src/ingress/inclusion_lane/config.rs +++ b/sequencer/src/ingress/inclusion_lane/config.rs @@ -7,8 +7,6 @@ use std::path::PathBuf; use std::time::Duration; -use alloy_primitives::Address; - const DEFAULT_MAX_USER_OPS_PER_CHUNK: usize = 64; const DEFAULT_SAFE_INPUT_BUFFER_CAPACITY: usize = 2048; const DEFAULT_MAX_BATCH_OPEN: Duration = Duration::from_secs(2 * 60 * 60); @@ -20,10 +18,6 @@ const DEFAULT_FRONTIER_MIN_INTERVAL: Duration = Duration::from_secs(1); #[derive(Debug, Clone)] pub struct InclusionLaneConfig { - /// Address of the batch submitter wallet. Direct inputs from this sender - /// are skipped during application execution (they're our own batch - /// submissions; the application doesn't apply them as user-level inputs). - pub batch_submitter_address: Address, /// Directory under which the lane creates snapshot dumps. Each dump /// lives in its own unique subdirectory of this path. The runtime /// sets this to `{data_dir}/dumps/`. @@ -45,9 +39,8 @@ pub struct InclusionLaneConfig { } impl InclusionLaneConfig { - pub fn new(batch_submitter_address: Address, dumps_dir: PathBuf) -> Self { + pub fn new(dumps_dir: PathBuf) -> Self { Self { - batch_submitter_address, dumps_dir, max_user_ops_per_chunk: DEFAULT_MAX_USER_OPS_PER_CHUNK, safe_input_buffer_capacity: DEFAULT_SAFE_INPUT_BUFFER_CAPACITY, diff --git a/sequencer/src/ingress/inclusion_lane/dump_info.rs b/sequencer/src/ingress/inclusion_lane/dump_info.rs index e143cf9a..8bcc7c6c 100644 --- a/sequencer/src/ingress/inclusion_lane/dump_info.rs +++ b/sequencer/src/ingress/inclusion_lane/dump_info.rs @@ -1,30 +1,8 @@ // (c) Cartesi and individual authors (see AUTHORS) // SPDX-License-Identifier: Apache-2.0 (see LICENSE) -//! Sequencer-owned dump-directory structure and metadata (`info.toml`). -//! -//! Every dump is a directory `dumps//` with exactly two entries: -//! -//! ```text -//! dumps// -//! state app-owned file or directory — the prefix handed to -//! `Application::{create_dump, from_dump}` -//! info.toml sequencer-owned checkpoint metadata (this module) -//! ``` -//! -//! `info.toml` makes a finalized dump a self-contained checkpoint for the -//! recovery handoff — and, together with the app's `state` -//! file or directory, the unit an operator backs up: `setup --recovery` rebuilds a wiped -//! DB from exactly this pair, reading `N` straight from the file. `next_batch_nonce` -//! (`N`) is known at batch close and written then; `promoted_inclusion_block` -//! (`B`) is known at promotion and stamped in place afterwards. An in-place update of a file -//! *inside* the dir changes no path, so the no-dangling-row invariant, leases, -//! and GC — all keyed on the immutable directory path — are untouched. -//! -//! The DB row (`dumps.prefix`) stores the dump *directory*; the app prefix is -//! always derived via [`app_prefix`]. Crash-safety ordering is unchanged from -//! the lifecycle doc: dir + `info.toml` + app dump are durable on disk before -//! the row that references the dir; row deletion precedes file deletion. +//! Immutable restore-artifact metadata. Accepted recovery exports add a +//! separate `checkpoint.toml` receipt selected from SQLite under a dump lease. use std::io; use std::path::{Path, PathBuf}; @@ -36,7 +14,7 @@ const APP_STATE_SUBDIR: &str = "state"; /// Name of the sequencer-owned metadata file inside a dump directory. const INFO_FILE: &str = "info.toml"; -pub const FORMAT_VERSION: u64 = 1; +pub const FORMAT_VERSION: u64 = 2; /// The app's dump prefix inside `dump_dir`. Pure path derivation. pub fn app_prefix(dump_dir: &Path) -> PathBuf { @@ -57,61 +35,89 @@ pub(crate) fn referenced_artifact_io_is_terminal(source: &io::Error) -> bool { ) } -/// Sequencer-owned checkpoint metadata for one dump. -/// -/// Serialized as real TOML (`#[serde(deny_unknown_fields)]` keeps the parse -/// strict — unknown keys, type mismatches, and TOML's own duplicate-key ban all -/// fail loud, the same guarantees the old hand parser gave). The on-disk layout -/// is plain `key = value` integer lines, so it is operator-readable and any -/// previously-written file round-trips unchanged. +/// Metadata known when the application artifact is created. #[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] #[serde(deny_unknown_fields)] pub struct DumpInfo { pub format_version: u64, - /// `N` — the batch nonce the sequencer resumes submitting at when - /// booting from this checkpoint (snapshot batch's nonce + 1; 0 for - /// the genesis dump). Known at batch close. pub next_batch_nonce: u64, - /// Replay cursor: global valid replay head at dump time. Mirrors the - /// snapshot row exactly. - pub l2_tx_index: u64, - /// `B` — the L1 inclusion block of the promotion that finalized this - /// dump. `None` until promotion; stamped in place when the batch is - /// observed accepted (and re-stamped from the DB at startup, closing - /// the commit-then-stamp crash window). Omitted from the file while - /// `None` (TOML has no null); a missing key reads back as `None`. - #[serde(skip_serializing_if = "Option::is_none")] - pub promoted_inclusion_block: Option, } impl DumpInfo { - /// Checkpoint metadata for the snapshot taken at the close of batch - /// `batch_nonce`: the sequencer resumes submitting at `batch_nonce + 1`, and - /// `B` is unknown until promotion (stamped in place then). The single home - /// for the resume-nonce `+ 1` skew, so the batch-close and recovery-fill - /// sites cannot drift on how `next_batch_nonce` is derived. - pub fn at_batch_close(batch_nonce: u64, l2_tx_index: u64) -> Self { + pub fn at_batch_close(batch_nonce: u64) -> Self { Self { format_version: FORMAT_VERSION, - next_batch_nonce: batch_nonce + 1, - l2_tx_index, - promoted_inclusion_block: None, + next_batch_nonce: batch_nonce.checked_add(1).expect("batch nonce overflow"), } } - /// Checkpoint metadata for a finalized snapshot rebuilt by `setup --recovery` - /// at inclusion block `inclusion_block`, resuming at `resume_nonce` — the - /// fold's next-expected nonce `N'`, already the resume value (no `+ 1`). - pub fn at_recovery(resume_nonce: u64, l2_tx_index: u64, inclusion_block: u64) -> Self { + pub fn at_baseline(next_batch_nonce: u64) -> Self { Self { format_version: FORMAT_VERSION, - next_batch_nonce: resume_nonce, - l2_tx_index, - promoted_inclusion_block: Some(inclusion_block), + next_batch_nonce, } } } +/// Acceptance receipt packaged with an operator's recovery export. Its block +/// is an exact end-of-block comparison boundary under per-batch snapshotting. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(deny_unknown_fields)] +pub struct CheckpointInfo { + pub format_version: u64, + pub next_batch_nonce: u64, + pub inclusion_block: u64, +} + +pub fn read_checkpoint_info(dump_dir: &Path) -> io::Result { + let content = std::fs::read_to_string(dump_dir.join("checkpoint.toml"))?; + let info: CheckpointInfo = + toml::from_str(&content).map_err(|err| io::Error::new(io::ErrorKind::InvalidData, err))?; + if info.format_version != FORMAT_VERSION { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "unsupported checkpoint format", + )); + } + Ok(info) +} + +/// Stream a complete, restorable artifact. A receipt makes it an accepted +/// recovery export; the on-disk dump remains unchanged. +pub(crate) fn write_archive( + writer: W, + dump_dir: &Path, + next_batch_nonce: u64, + checkpoint: Option<&CheckpointInfo>, +) -> io::Result<()> { + let info = read_info(dump_dir)?; + if info.next_batch_nonce != next_batch_nonce { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "dump nonce differs from snapshot boundary", + )); + } + let mut archive = tar::Builder::new(writer); + archive.append_path_with_name(dump_dir.join(INFO_FILE), INFO_FILE)?; + let state = app_prefix(dump_dir); + if state.is_dir() { + archive.append_dir_all(APP_STATE_SUBDIR, state)?; + } else { + archive.append_path_with_name(state, APP_STATE_SUBDIR)?; + } + if let Some(checkpoint) = checkpoint { + let bytes = toml::to_string(checkpoint) + .map_err(|err| io::Error::new(io::ErrorKind::InvalidData, err))? + .into_bytes(); + let mut header = tar::Header::new_gnu(); + header.set_size(bytes.len() as u64); + header.set_mode(0o600); + header.set_cksum(); + archive.append_data(&mut header, "checkpoint.toml", bytes.as_slice())?; + } + archive.finish() +} + /// Errors from creating a structured dump directory. #[derive(Debug, thiserror::Error)] pub enum CreateDumpDirError { @@ -148,8 +154,7 @@ pub fn delete_dump_dir(dump_dir: &Path) -> io::Result<()> { } /// Write `info.toml` into `dump_dir`, durably: temp file, fsync, rename -/// over, fsync the directory. Safe both for initial creation and for the -/// in-place promotion stamp. +/// over, fsync the directory. Called only during artifact creation. pub fn write_info(dump_dir: &Path, info: &DumpInfo) -> io::Result<()> { let content = toml::to_string(info) .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, format!("info.toml: {e}")))?; @@ -206,8 +211,8 @@ pub fn read_info(dump_dir: &Path) -> io::Result { /// (`manifest.json` + `snapshot/`) from a merely empty/wrong path. pub fn diagnose_missing_dump(dump_dir: &Path) -> String { let expected = format!( - "expected a finalized sequencer dump at {} with `info.toml` and a `state` \ - app dump (usually `$CARTESI_SEQUENCER_DATA_DIR/dumps//`). \ + "expected a sequencer dump at {} with `info.toml` and a `state` app artifact. \ + Recovery additionally requires `checkpoint.toml` from `/finalized_snapshot`. \ See docs/snapshots/lifecycle.md and docs/recovery/cockroach.md.", dump_dir.display() ); @@ -246,17 +251,6 @@ pub fn diagnose_missing_dump(dump_dir: &Path) -> String { format!("missing `info.toml` — {expected}") } -/// Stamp `B` (the promotion's inclusion block) into an existing -/// `info.toml`. Idempotent: re-stamping the same block is a no-op write. -pub fn stamp_promoted_inclusion_block(dump_dir: &Path, block: u64) -> io::Result<()> { - let mut info = read_info(dump_dir)?; - if info.promoted_inclusion_block == Some(block) { - return Ok(()); - } - info.promoted_inclusion_block = Some(block); - write_info(dump_dir, &info) -} - #[cfg(test)] mod tests { use super::*; @@ -413,17 +407,49 @@ mod tests { assert_dump_prefix_round_trip::(); } + #[test] + fn archives_restore_file_and_directory_artifacts_without_mutating_metadata() { + fn check() { + let root = tempfile::tempdir().unwrap(); + let source = root.path().join("source"); + let mut app = PrefixDumpApp::::default(); + create_dump_dir_with_info(&mut app, &source, &sample()).unwrap(); + let original_info = std::fs::read(source.join(INFO_FILE)).unwrap(); + let receipt = CheckpointInfo { + format_version: FORMAT_VERSION, + next_batch_nonce: 7, + inclusion_block: 42, + }; + let mut bytes = Vec::new(); + write_archive(&mut bytes, &source, 7, Some(&receipt)).unwrap(); + assert_eq!( + std::fs::read(source.join(INFO_FILE)).unwrap(), + original_info + ); + assert!(!source.join("checkpoint.toml").exists()); + delete_dump_dir(&source).unwrap(); + let destination = root.path().join("restored"); + tar::Archive::new(bytes.as_slice()) + .unpack(&destination) + .unwrap(); + assert_eq!(read_checkpoint_info(&destination).unwrap(), receipt); + let restored = + PrefixDumpApp::::from_dump(&app_prefix(&destination)).unwrap(); + assert_eq!(restored.progress(), app.progress()); + } + check::(); + check::(); + } + fn sample() -> DumpInfo { DumpInfo { format_version: FORMAT_VERSION, next_batch_nonce: 7, - l2_tx_index: 123, - promoted_inclusion_block: None, } } #[test] - fn write_then_read_round_trips_without_promotion() { + fn immutable_info_round_trips() { let dir = tempfile::tempdir().unwrap(); write_info(dir.path(), &sample()).unwrap(); assert_eq!(read_info(dir.path()).unwrap(), sample()); @@ -449,23 +475,6 @@ mod tests { ); } - #[test] - fn stamp_fills_b_and_is_idempotent() { - let dir = tempfile::tempdir().unwrap(); - write_info(dir.path(), &sample()).unwrap(); - - stamp_promoted_inclusion_block(dir.path(), 456).unwrap(); - let stamped = read_info(dir.path()).unwrap(); - assert_eq!(stamped.promoted_inclusion_block, Some(456)); - assert_eq!(stamped.next_batch_nonce, 7, "other fields untouched"); - - stamp_promoted_inclusion_block(dir.path(), 456).unwrap(); - assert_eq!( - read_info(dir.path()).unwrap().promoted_inclusion_block, - Some(456) - ); - } - #[test] fn read_rejects_unknown_duplicate_and_missing_keys() { let dir = tempfile::tempdir().unwrap(); @@ -484,23 +493,6 @@ mod tests { assert!(read_info(dir.path()).is_err(), "missing keys must reject"); } - #[test] - fn on_disk_bytes_stay_simple_key_value_lines() { - // The serialized form must remain plain `key = value` integer lines: - // no `[table]` headers, no quoting. This is the operator-readable - // contract and what any previously-written file already looks like. - let dir = tempfile::tempdir().unwrap(); - let mut info = sample(); - info.promoted_inclusion_block = Some(456); - write_info(dir.path(), &info).unwrap(); - let bytes = std::fs::read_to_string(dir.path().join(INFO_FILE)).unwrap(); - assert_eq!( - bytes, - "format_version = 1\nnext_batch_nonce = 7\nl2_tx_index = 123\n\ - promoted_inclusion_block = 456\n" - ); - } - #[test] fn read_tolerates_comments_blanks_and_reordering() { // The robustness win from real TOML over the old line parser: an @@ -509,8 +501,7 @@ mod tests { let dir = tempfile::tempdir().unwrap(); std::fs::write( dir.path().join(INFO_FILE), - "# checkpoint metadata\n\nl2_tx_index = 123\nnext_batch_nonce = 7\n\ - format_version = 1\n", + "# checkpoint metadata\n\nnext_batch_nonce = 7\nformat_version = 2\n", ) .unwrap(); assert_eq!(read_info(dir.path()).unwrap(), sample()); @@ -528,7 +519,8 @@ mod tests { "got: {msg}" ); assert!(msg.contains("manifest.json and snapshot/"), "got: {msg}"); - assert!(msg.contains("dumps//"), "got: {msg}"); + assert!(msg.contains("/finalized_snapshot"), "got: {msg}"); + assert!(msg.contains("checkpoint.toml"), "got: {msg}"); let err = read_info(dir.path()).unwrap_err(); assert_eq!(err.kind(), io::ErrorKind::NotFound); @@ -566,7 +558,7 @@ mod tests { let dir = tempfile::tempdir().unwrap(); std::fs::write( dir.path().join(INFO_FILE), - "format_version = 999\nnext_batch_nonce = 0\nl2_tx_index = 0\n", + "format_version = 999\nnext_batch_nonce = 0\n", ) .unwrap(); assert!(read_info(dir.path()).is_err()); diff --git a/sequencer/src/ingress/inclusion_lane/error.rs b/sequencer/src/ingress/inclusion_lane/error.rs index 5f1a7781..74d4331a 100644 --- a/sequencer/src/ingress/inclusion_lane/error.rs +++ b/sequencer/src/ingress/inclusion_lane/error.rs @@ -8,7 +8,7 @@ use sequencer_core::application::AppError; use thiserror::Error; use super::dump_info::CreateDumpDirError; -use super::snapshot::{GcError, StampError, TakeDumpError}; +use super::snapshot::{GcError, TakeDumpError}; #[derive(Debug, Error)] pub enum InclusionLaneError { @@ -42,8 +42,6 @@ pub enum InclusionLaneError { LoadFromDump(AppError), #[error("snapshot garbage collection failed")] Gc(#[from] GcError), - #[error("stamping promotion metadata into the finalized dump failed")] - PromotionStamp(#[from] StampError), #[error( "no open Tip at lane startup; the runtime must establish it via \ guarded startup recovery before starting the lane" @@ -61,12 +59,11 @@ impl InclusionLaneError { } Self::LoadFromDump(source) => referenced_snapshot_app_error_is_terminal(source), Self::Snapshot(source) => take_dump_error_is_terminal(source), - Self::Gc(GcError::Storage(source)) - | Self::PromotionStamp(StampError::Storage(source)) => { + Self::Gc(GcError::Storage(source)) => { crate::storage::is_persistent_storage_error(source) } Self::CanonicalDivergence { .. } | Self::NoOpenTip => true, - Self::ChannelClosed | Self::PromotionStamp(StampError::Io(_)) => false, + Self::ChannelClosed => false, } } } @@ -92,6 +89,8 @@ fn take_dump_error_is_terminal(source: &TakeDumpError) -> bool { #[derive(Debug, Error)] pub enum CatchUpError { + #[error(transparent)] + History(#[from] sequencer_core::history::HistoryPolicyError), #[error("cannot load resume snapshot")] LoadSnapshot { #[source] @@ -115,15 +114,6 @@ pub enum CatchUpError { }, #[error("snapshot executed-input count mismatch: application={application}, storage={storage}")] SnapshotExecutionCountMismatch { application: u64, storage: u64 }, - #[error( - "physical replay row {db_offset} ({kind}) has execution offset {stored:?}, expected {expected:?}" - )] - ExecutionOffsetMismatch { - db_offset: u64, - kind: &'static str, - expected: Option, - stored: Option, - }, #[error( "no snapshot registered before lane catch-up; \ runtime must ensure a genesis dump exists at first startup" @@ -140,9 +130,9 @@ impl CatchUpError { Self::ReplayUserOp { source } | Self::ReplayDirectInput { source } => { app_error_is_terminal(source) } - Self::NoSnapshot - | Self::SnapshotExecutionCountMismatch { .. } - | Self::ExecutionOffsetMismatch { .. } => true, + Self::History(_) | Self::NoSnapshot | Self::SnapshotExecutionCountMismatch { .. } => { + true + } } } } @@ -213,9 +203,6 @@ mod tests { InclusionLaneError::Snapshot(TakeDumpError::CreateDump(CreateDumpDirError::Io( std::io::Error::other("dump directory unavailable"), ))), - InclusionLaneError::PromotionStamp(StampError::Io(std::io::Error::other( - "metadata unavailable", - ))), ]; for error in errors { @@ -242,9 +229,6 @@ mod tests { rusqlite::Error::QueryReturnedNoRows, )), InclusionLaneError::Gc(GcError::Storage(rusqlite::Error::QueryReturnedNoRows)), - InclusionLaneError::PromotionStamp(StampError::Storage( - rusqlite::Error::QueryReturnedNoRows, - )), ]; for error in errors { diff --git a/sequencer/src/ingress/inclusion_lane/mod.rs b/sequencer/src/ingress/inclusion_lane/mod.rs index b60bc432..685e3d0a 100644 --- a/sequencer/src/ingress/inclusion_lane/mod.rs +++ b/sequencer/src/ingress/inclusion_lane/mod.rs @@ -11,12 +11,11 @@ //! subset commits at most once, bounding ack latency for its first op. //! All-rejected chunks mutate nothing and do not open a transaction. //! - **L1 reconciliation** (observed at `frontier_min_interval`): once five -//! newly-safe blocks have accumulated, consumes the complete range, promotes -//! snapshots, and advances one frame directly to the observed tip. The time +//! newly-safe blocks have accumulated, executes the complete direct-input range and advances one frame directly to the observed tip. The time //! gate bounds SQL load; block distance is the semantic clock criterion. //! That frontier read is also the lane's divergence refusal point (I15): //! a marker already present closes intake before direct execution, -//! promotion, or the frame-clock decision. +//! or the frame-clock decision. //! //! The lane is a single-thread `spawn_blocking` task. SQLite is the durable data //! coordination boundary with the input reader and batch submitter. HTTP @@ -47,11 +46,11 @@ use tokio::sync::mpsc; use tokio::task::JoinHandle; use crate::runtime::shutdown::RuntimeScope; -use crate::storage::{SafeFrontierState, SafeInputRange, Storage, StoredSafeInput, WriteHead}; +use crate::storage::{SafeFrontierState, SafeInputRange, Storage, StoredDirectInput, WriteHead}; use sequencer_core::application::{ Application, ExecutionOutcome, execute_direct_input, validate_and_execute_user_op, }; -use sequencer_core::l2_tx::DirectInput; +use sequencer_core::history::ExecutedInputCount; use sequencer_core::user_op::SignedUserOp; use catch_up::{catch_up_application, catch_up_snapshot}; @@ -72,14 +71,9 @@ impl InclusionLane { /// It fails loudly with /// [`InclusionLaneError::NoOpenTip`] if the invariant was somehow violated. /// - /// The lane selects one resume checkpoint — the latest pending - /// snapshot if any, else the finalized snapshot (see - /// `catch_up_snapshot`) — and uses it for both `A::from_dump` and the - /// catch-up replay offset, so the loaded state and the replay cursor - /// can never drift apart. The runtime guarantees at least a genesis - /// finalized snapshot exists before this is called (cold start - /// registers one; warm start reuses the previous run's). A missing - /// snapshot surfaces as `CatchUpError::NoSnapshot`. + /// Restore the newest surviving snapshot and replay from its exact + /// application count. Startup preserves a rollback-safe checkpoint before + /// admitting any runtime work. /// /// Returns the input MPSC sender (for the API to enqueue user /// ops) and the join handle (for the runtime to observe lane @@ -114,7 +108,7 @@ impl InclusionLane { }); } tracing::debug!( - l2_tx_index = checkpoint.l2_tx_index, + executed_input_count = checkpoint.executed_input_count.get(), "inclusion lane resuming from snapshot" ); let mut lane = Self { @@ -124,12 +118,12 @@ impl InclusionLane { storage, config, }; - lane.run_forever(checkpoint.l2_tx_index) + lane.run_forever(checkpoint.executed_input_count) }); (tx, handle) } - fn run_forever(&mut self, catch_up_from: u64) -> Result<(), InclusionLaneError> { + fn run_forever(&mut self, catch_up_from: ExecutedInputCount) -> Result<(), InclusionLaneError> { self.run_catch_up(catch_up_from)?; let mut included = Vec::with_capacity(self.config.max_user_ops_per_chunk.max(1)); let mut safe_inputs = Vec::with_capacity(self.config.safe_input_buffer_capacity.max(1)); @@ -159,9 +153,9 @@ impl InclusionLane { { let next_safe_block = lane_state.head.safe_block; // Atomic close: dump the app state, then seal the batch - // and register its pending snapshot in one transaction. + // and register its snapshot in one transaction. // A create_dump failure leaves the batch open for retry; - // a committed close always has a promotable snapshot row. + // a committed close always has its immutable snapshot row. // Errors propagate per the lane's fail-loud policy. snapshot::close_batch_with_snapshot( &mut self.app, @@ -172,23 +166,15 @@ impl InclusionLane { ) .map_err(InclusionLaneError::Snapshot)?; } else if !turn.processed_any() { - // Nothing to drain and no batch to close: back off. GC no longer - // lives here — it runs after a promotion in - // `maybe_advance_safe_frontier`, so it tracks garbage creation - // rather than idleness and is never starved under load. + // Nothing to drain and no batch to close: back off. GC runs on the reconciliation cadence so load cannot starve it. thread::sleep(self.config.idle_poll_interval); } } } - fn run_catch_up(&mut self, start_offset: u64) -> Result<(), InclusionLaneError> { - catch_up_application( - &mut self.app, - &mut self.storage, - self.config.batch_submitter_address, - start_offset, - ) - .map_err(|source| InclusionLaneError::CatchUp { source }) + fn run_catch_up(&mut self, start_offset: ExecutedInputCount) -> Result<(), InclusionLaneError> { + catch_up_application(&mut self.app, &mut self.storage, start_offset) + .map_err(|source| InclusionLaneError::CatchUp { source }) } /// Process at most one bounded dequeue chunk. Returning to the outer loop @@ -225,7 +211,7 @@ impl InclusionLane { fn maybe_advance_safe_frontier( &mut self, lane_state: &mut LaneState, - safe_inputs: &mut Vec, + safe_inputs: &mut Vec, ) -> Result<(), InclusionLaneError> { if !lane_state.frontier_check_due(self.config.frontier_min_interval) { return Ok(()); @@ -265,100 +251,42 @@ impl InclusionLane { let leading_direct_range = SafeInputRange::new(lane_state.next_safe_input_index, frontier.end_exclusive); - // The observation commits its promotion (if any) in the same - // transaction as the drain, so a crash can never leave a - // promoted-but-undrained batch — the state a restart would re-process - // and re-promote on a deleted pending row. - let observation = self.execute_safe_inputs_range(leading_direct_range, safe_inputs)?; - let promoted = observation.commit( - &mut self.storage, + let executions = self.execute_direct_range(leading_direct_range, safe_inputs)?; + self.storage.close_frame_only_with_executions( &mut lane_state.head, frontier.safe_block, leading_direct_range, + &executions, )?; lane_state.next_safe_input_index = frontier.end_exclusive; - // A promotion supersedes the previous finalized (and any lower-nonce - // pendings); reclaim them now. The full pass also collects earlier - // lease-released garbage. On the lane's own thread, only when a - // promotion created garbage — so GC tracks garbage creation, never - // starved by load. - if promoted { - // Stamp `B` into the freshly finalized dump's info.toml before - // GC (the stamp targets the survivor; GC removes the superseded). - snapshot::stamp_finalized_promotion(&mut self.storage)?; - let removed = snapshot::run_gc(&mut self.storage).map_err(InclusionLaneError::Gc)?; - if removed > 0 { - tracing::debug!(removed, "post-promotion GC removed unreferenced dumps"); - } + // Acceptance can advance independently of application inputs. A bounded + // reconciliation cadence reclaims retired artifacts even under load. + let removed = snapshot::run_gc(&mut self.storage)?; + if removed > 0 { + tracing::debug!(removed, "snapshot garbage collection removed artifacts"); } Ok(()) } - /// Process the safe inputs in `direct_range`, accumulating which of our - /// batches landed into a [`snapshot::BlockObservation`] for the caller to - /// [`commit`](snapshot::BlockObservation::commit). - fn execute_safe_inputs_range( - &mut self, - direct_range: SafeInputRange, - chunk: &mut Vec, - ) -> Result { - let mut observation = snapshot::BlockObservation::new(); - let max_chunk_len = self.config.safe_input_buffer_capacity.max(1) as u64; - for chunk_range in direct_range.chunks(max_chunk_len) { - self.storage.fill_safe_inputs(chunk_range, chunk)?; - self.execute_safe_inputs_chunk( - chunk.as_slice(), - chunk_range.start(), - &mut observation, - )?; - } - Ok(observation) - } - - fn execute_safe_inputs_chunk( + fn execute_direct_range( &mut self, - chunk: &[StoredSafeInput], - base_safe_input_index: u64, - observation: &mut snapshot::BlockObservation, - ) -> Result<(), InclusionLaneError> { - for (offset, input) in chunk.iter().enumerate() { - let safe_input_index = base_safe_input_index + offset as u64; - let own_batch_nonce = if input.sender == self.config.batch_submitter_address { - // Look up whether the scheduler accepted this batch. - // Stale-nonce batches end up in safe_inputs but NOT in - // safe_accepted_batches; only accepted ones get promoted. - self.storage - .accepted_batch_nonce_at(safe_input_index) - .map_err(InclusionLaneError::Storage)? - } else { - None - }; - - // Accumulate the observation — infallible, no storage. The lane - // promotes once at range close, atomically with the drain. - observation.observe(input.block_number, own_batch_nonce); - - if input.sender == self.config.batch_submitter_address { - // Our own batch (accepted or rejected) — never replayed - // as a direct input. - continue; + range: SafeInputRange, + chunk: &mut Vec, + ) -> Result, InclusionLaneError> { + let mut executions = Vec::new(); + for chunk_range in range.chunks(self.config.safe_input_buffer_capacity.max(1) as u64) { + self.storage.fill_direct_inputs(chunk_range, chunk)?; + for input in chunk.iter() { + let receipt = execute_direct_input(&mut self.app, &input.input) + .map_err(|source| InclusionLaneError::ExecuteDirectInput { source })?; + executions.push(crate::storage::DirectInputExecution { + safe_input_index: input.safe_input_index, + executed_input_offset: receipt.offset, + }); } - - let direct_input = DirectInput { - sender: input.sender, - block_number: input.block_number, - payload: input.payload.clone(), - }; - - let receipt = execute_direct_input(&mut self.app, &direct_input) - .map_err(|source| InclusionLaneError::ExecuteDirectInput { source })?; - observation.observe_direct_execution(crate::storage::DirectInputExecution { - safe_input_index, - executed_input_offset: receipt.offset, - }); } - Ok(()) + Ok(executions) } fn respond_internal_to_all(pending: &mut Vec, message: String) { diff --git a/sequencer/src/ingress/inclusion_lane/snapshot.rs b/sequencer/src/ingress/inclusion_lane/snapshot.rs index b3ce78ee..3542eb5f 100644 --- a/sequencer/src/ingress/inclusion_lane/snapshot.rs +++ b/sequencer/src/ingress/inclusion_lane/snapshot.rs @@ -1,44 +1,16 @@ // (c) Cartesi and individual authors (see AUTHORS) // SPDX-License-Identifier: Apache-2.0 (see LICENSE) -//! Snapshot lifecycle integration for the inclusion lane. Three -//! responsibilities (see `docs/snapshots/lifecycle.md` for the full design): -//! -//! 1. **At batch close**, call [`close_batch_with_snapshot`]: dump the -//! live Application's state, then seal the batch and register the -//! dump in `pending_snapshots` (keyed by the batch's nonce) in one -//! transaction. The atomic seal+register guarantees every sealed -//! batch has a promotable snapshot row. Errors propagate through the -//! lane's exit per the fail-loud policy. -//! -//! 2. **While processing safe inputs**, thread a [`BlockObservation`] -//! through `execute_safe_inputs_chunk` to accumulate the highest-nonce -//! batch of ours that landed in the range. At range close the lane -//! promotes that one `(nonce, block)` target, folded into the same -//! transaction that advances the drain -//! ([`crate::storage::Storage::close_frame_only_with_executions`]) -//! — so promotion, drain, and canonical execution attributions commit -//! atomically. Promotion is **per-range, not per-block**: the range's max -//! nonce supersedes every lower one, and the skipped intermediate -//! checkpoints were never observable. -//! -//! 3. **After a promotion**, the lane runs [`run_gc`] to reclaim the -//! now-superseded dump(s). GC tracks garbage creation, not idleness. -//! -//! The observer holds one `Option<(nonce, block)>` plus direct-execution -//! receipts for the complete range. That allocation is confined to the slow -//! L1-reconciliation regime, not the user-op hot path. +//! Durable batch-close artifacts and filesystem cleanup after SQLite GC. use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicU64, Ordering}; use std::time::{SystemTime, UNIX_EPOCH}; -use sequencer_core::application::Application; - use super::dump_info::{self, DumpInfo}; -use crate::storage::{DirectInputExecution, SafeInputRange, Storage, WriteHead}; +use crate::storage::{Storage, WriteHead}; +use sequencer_core::application::Application; -/// Errors from snapshot-taking at batch close. #[derive(Debug, thiserror::Error)] pub enum TakeDumpError { #[error("storage: {0}")] @@ -47,82 +19,25 @@ pub enum TakeDumpError { CreateDump(#[from] dump_info::CreateDumpDirError), } -/// Errors from the post-promotion GC pass. #[derive(Debug, thiserror::Error)] pub enum GcError { #[error("storage: {0}")] Storage(#[from] rusqlite::Error), } -/// Errors from stamping promotion metadata (`B`) into the finalized -/// dump's `info.toml`. -#[derive(Debug, thiserror::Error)] -pub enum StampError { - #[error("storage: {0}")] - Storage(#[from] rusqlite::Error), - #[error("io: {0}")] - Io(#[from] std::io::Error), -} - -/// Run one garbage-collection pass: delete every unreferenced dump -/// row in SQLite (atomically) and best-effort delete the corresponding -/// directories on disk. Filesystem failures log and continue — an -/// orphan file is acceptable per the no-dangling-row invariant; only -/// the reverse (SQLite row pointing at a missing path) would matter, -/// and the SQL-first ordering prevents it. pub(super) fn run_gc(storage: &mut Storage) -> Result { let removed = storage.gc_unreferenced_dumps()?; for row in &removed { if let Err(err) = dump_info::delete_dump_dir(&row.prefix) { - tracing::warn!( - error = %err, - prefix = ?row.prefix, - "GC: filesystem delete failed; orphan left for next startup sweep", - ); + tracing::warn!(error = %err, prefix = ?row.prefix, + "GC: filesystem delete failed; orphan left for next startup sweep"); } } Ok(removed.len()) } -/// Stamp `B` — the promotion's L1 inclusion block — into the freshly -/// finalized dump's `info.toml`, completing the checkpoint metadata -/// whose other fields were written at batch close. During live operation -/// the DB row is authoritative for `B` and `info.toml` mirrors it (a -/// crash between the promoting commit and this stamp is healed by the -/// idempotent startup re-stamp from the same row). But `info.toml` is -/// also the durable checkpoint the operator backs up and `setup -/// --recovery` reads — there the DB is gone and the file is the sole -/// authority for the resume nonce `N` — so keeping the two coherent here -/// is load-bearing, not cosmetic. -pub(super) fn stamp_finalized_promotion(storage: &mut Storage) -> Result<(), StampError> { - let finalized = storage - .finalized_dump()? - .expect("a promotion just committed, so the finalized snapshot row exists"); - dump_info::stamp_promoted_inclusion_block(&finalized.dump.prefix, finalized.inclusion_block)?; - Ok(()) -} - -/// Close the current batch and register its snapshot atomically. -/// -/// Order matters for crash/error safety: -/// 1. Read the closing batch's nonce (assigned at open) and the replay -/// head; build a unique dump directory. -/// 2. [`dump_info::create_dump_dir_with_info`] writes + fsyncs the dir, -/// `info.toml` (`N` = nonce + 1, the replay head; `B` stamped later -/// at promotion), and the app's dump — all **before** any DB -/// mutation. On failure nothing is sealed — the batch stays the open -/// Tip; the error propagates per the lane's fail-loud policy (the -/// retry happens on the next boot, after catch-up). -/// 3. One DB transaction seals the batch, opens the next, and inserts -/// the `pending_snapshots` row -/// ([`Storage::close_frame_and_batch_with_pending_dump`]). A -/// committed close therefore always has a promotable snapshot; a tx -/// failure rolls the seal back, leaving only an orphan dump -/// directory (reaped by the startup sweep). -/// -/// Errors propagate to the lane's main loop per the fail-loud policy — -/// a chronic snapshot failure (e.g. a full disk) is an operational -/// problem to surface, not paper over. +/// Files become durable before the atomic batch seal and snapshot registration. +/// A failed commit leaves only an orphan directory for the startup sweep. pub(super) fn close_batch_with_snapshot( app: &mut A, storage: &mut Storage, @@ -130,137 +45,20 @@ pub(super) fn close_batch_with_snapshot( next_safe_block: u64, dumps_dir: &Path, ) -> Result<(), TakeDumpError> { - let nonce = storage.batch_nonce(head.batch_index)?; - // The snapshot reflects state through the global valid replay head - // as of the close; single-writer lane, so the head can't move before - // the close transaction (which re-asserts equality). - let l2_tx_index = storage.valid_ordered_l2_tx_head()?; + let batch_index = head.batch_index; + let nonce = storage.batch_nonce(batch_index)?; let dump_dir = make_dump_dir(dumps_dir, nonce); - dump_info::create_dump_dir_with_info( - app, - &dump_dir, - &DumpInfo::at_batch_close(nonce, l2_tx_index), - )?; - storage.close_frame_and_batch_with_pending_dump( + dump_info::create_dump_dir_with_info(app, &dump_dir, &DumpInfo::at_batch_close(nonce))?; + storage.close_frame_and_batch_with_snapshot( head, next_safe_block, &dump_dir, - nonce, - l2_tx_index, + batch_index, app.executed_input_count(), )?; Ok(()) } -/// Register a pending snapshot for an **already-closed** batch (test -/// helper for seeding `pending_snapshots`). Production closes batches -/// via [`close_batch_with_snapshot`], which is atomic; this two-step -/// "dump then insert" shape only exists so tests can stage pending rows -/// against batches sealed by `seed_closed_batches`. -#[cfg(test)] -pub(super) fn take_dump_at_batch_close( - app: &mut A, - storage: &mut Storage, - dumps_dir: &Path, - closed_batch_index: u64, -) -> Result<(), TakeDumpError> { - let nonce = storage.batch_nonce(closed_batch_index)?; - // The snapshot reflects state through the global valid replay head, - // not just this batch's own rows — so an empty batch correctly - // records the prior head rather than genesis. - let l2_tx_index = storage.valid_ordered_l2_tx_head()?; - let dump_dir = make_dump_dir(dumps_dir, nonce); - dump_info::create_dump_dir_with_info( - app, - &dump_dir, - &DumpInfo::at_batch_close(nonce, l2_tx_index), - )?; - storage.insert_pending_dump(&dump_dir, nonce, l2_tx_index)?; - Ok(()) -} - -/// Accumulator for batch promotion across one safe-input processing range. -/// -/// Records the highest accepted-batch nonce observed in the range and the L1 -/// block it landed in, then **commits itself once** at range close (via -/// [`BlockObservation::commit`]): the promotion, if any, folds into the same -/// transaction that advances the drain -/// ([`Storage::close_frame_only_with_executions`]) — so promotion, -/// drain, and canonical execution mappings commit atomically. -/// -/// Per-range (not per-block) promotion is sound because nonces land in -/// monotonic order: the range's max nonce sits in its latest -/// block-with-our-batch and supersedes every lower one (`promote_finalized` -/// deletes all pending `<= max`). Intermediate per-block checkpoints were never -/// observable anyway — `finalized` is a single async-polled row — so collapsing -/// to one promotion loses nothing. -/// -/// The accepted-batch observation is constant-sized. Direct execution receipts -/// are retained for the range so the eventual frame transaction can attach -/// their canonical offsets atomically; this allocation is confined to the -/// slow L1-reconciliation regime. -pub(super) struct BlockObservation { - /// `(nonce, inclusion_block)` of the highest accepted batch seen, or - /// `None` if the range observed none of our batches. - max: Option<(u64, u64)>, - direct_executions: Vec, -} - -impl BlockObservation { - pub(super) fn new() -> Self { - Self { - max: None, - direct_executions: Vec::new(), - } - } - - /// Record a safe input belonging to L1 block `block`; if it was one of our - /// accepted batches, with `nonce`. Keeps the highest nonce and its block. - pub(super) fn observe(&mut self, block: u64, own_batch_nonce: Option) { - if let Some(nonce) = own_batch_nonce { - // Monotonic landing order ⇒ a higher nonce is in a later-or-equal - // block, so the max nonce's block is the latest block-with-our-batch. - if self.max.is_none_or(|(max_nonce, _)| nonce > max_nonce) { - self.max = Some((nonce, block)); - } - } - } - - pub(super) fn observe_direct_execution(&mut self, execution: DirectInputExecution) { - self.direct_executions.push(execution); - } - - /// Close the frame for this safe-frontier advance, folding the observed - /// promotion — if any — into the **same transaction** as the drain - /// ([`Storage::close_frame_only_with_executions`]); otherwise an - /// attributed plain frame close. - /// Returns whether a batch was promoted, so the caller can collect the dumps - /// it superseded. Consumes the observation — it is spent once committed. - pub(super) fn commit( - self, - storage: &mut Storage, - head: &mut WriteHead, - next_safe_block: u64, - drained: SafeInputRange, - ) -> Result { - storage.close_frame_only_with_executions( - head, - next_safe_block, - drained, - &self.direct_executions, - self.max, - )?; - Ok(self.max.is_some()) - } - - /// Test-only inspection of the accumulated `(max_nonce, inclusion_block)`. - /// Production drives promotion through [`commit`](Self::commit). - #[cfg(test)] - pub(super) fn promotion(&self) -> Option<(u64, u64)> { - self.max - } -} - fn make_dump_dir(dumps_dir: &Path, nonce: u64) -> PathBuf { // Unique per call within a process: nonce + nanos + atomic counter. // Nonces can be reused across recovery cascades, so they alone @@ -275,340 +73,3 @@ fn make_dump_dir(dumps_dir: &Path, nonce: u64) -> PathBuf { .unwrap_or(0); dumps_dir.join(format!("nonce-{nonce}-{nanos}-{counter}")) } - -#[cfg(test)] -mod tests { - use std::path::{Path, PathBuf}; - - use alloy_primitives::Address; - use sequencer_core::application::{ - AppError, AppOutputs, Application, ApplicationProgress, ValidationOutcome, - }; - use sequencer_core::l2_tx::ValidUserOp; - use sequencer_core::user_op::UserOp; - - use crate::storage::Storage; - use crate::storage::test_helpers::{seed_closed_batches, temp_db}; - - use super::dump_info; - use super::{BlockObservation, take_dump_at_batch_close}; - - /// Minimal Application that records every `create_dump` call. The - /// other trait methods aren't exercised in these tests. - struct RecordingDumpApp { - dumps: Vec, - progress: ApplicationProgress, - } - - impl RecordingDumpApp { - fn new() -> Self { - Self { - dumps: Vec::new(), - progress: ApplicationProgress::default(), - } - } - - fn recorded(&self) -> Vec { - self.dumps.clone() - } - } - - impl Application for RecordingDumpApp { - fn max_method_payload_bytes() -> usize { - 0 - } - - fn validate_user_op( - &self, - _sender: Address, - _user_op: &UserOp, - _current_fee: u16, - ) -> Result { - Ok(ValidationOutcome::Accept) - } - - fn apply_valid_user_op( - &mut self, - _user_op: &ValidUserOp, - safe_block: u64, - ) -> Result { - self.progress.advance(safe_block); - Ok(Vec::new()) - } - - fn apply_direct_input( - &mut self, - _input: &sequencer_core::l2_tx::DirectInput, - ) -> Result { - unimplemented!("not used in these tests") - } - - fn progress(&self) -> ApplicationProgress { - self.progress - } - - fn from_dump(_prefix: &Path) -> Result { - unimplemented!("not used in these tests") - } - - fn create_dump(&mut self, prefix: &Path) -> Result<(), AppError> { - std::fs::create_dir(prefix)?; - std::fs::write(prefix.join("state"), b"recorded")?; - self.dumps.push(prefix.to_path_buf()); - Ok(()) - } - - fn state_file_in_dump(prefix: &Path) -> PathBuf { - prefix.join("state") - } - } - - /// Open a storage with `count` closed batches (indices 0..count-1, - /// nonces 0..count-1) using the public API. These batches have no - /// sequenced txs, so the global valid replay head is 0 and the - /// recorded `l2_tx_index` is 0. That's fine for these tests; the - /// l2_tx_index propagation through the snapshot lifecycle is - /// covered by storage-layer tests. - fn temp_storage_with_closed_batches( - name: &str, - count: u64, - ) -> (Storage, crate::storage::test_helpers::TestDb) { - let db = temp_db(name); - let mut storage = Storage::open(db.path.as_str()).expect("open"); - seed_closed_batches(&mut storage, count); - (storage, db) - } - - #[test] - fn take_dump_at_batch_close_creates_dump_and_pending_row() { - let (mut storage, _db) = temp_storage_with_closed_batches("take-dump", 3); - let dumps_dir = tempfile::tempdir().unwrap(); - let mut app = RecordingDumpApp::new(); - - // Batch index 1 is closed with nonce 1 (per seed_closed_batches). - take_dump_at_batch_close(&mut app, &mut storage, dumps_dir.path(), 1).unwrap(); - - let recorded = app.recorded(); - assert_eq!(recorded.len(), 1); - let app_prefix = &recorded[0]; - assert!(app_prefix.starts_with(dumps_dir.path())); - assert!( - app_prefix.join("state").exists(), - "dump's state file exists" - ); - - let pending = storage.latest_pending_dump().unwrap().unwrap(); - assert_eq!(pending.nonce, 1); - // The DB row stores the dump dir; the app dumped into `state`. - assert_eq!(dump_info::app_prefix(&pending.dump.prefix), *app_prefix); - assert_eq!(pending.l2_tx_index, 0); // empty batch → no L2 txs - - // info.toml carries the resume nonce and the same cursor; - // B is unstamped until promotion. - let info = dump_info::read_info(&pending.dump.prefix).unwrap(); - assert_eq!(info.next_batch_nonce, 2); - assert_eq!(info.l2_tx_index, 0); - assert_eq!(info.promoted_inclusion_block, None); - } - - #[test] - fn block_observation_keeps_the_max_nonce_and_its_block() { - let mut obs = BlockObservation::new(); - obs.observe(500, None); // not one of our batches - obs.observe(500, Some(0)); // our batch nonce 0, block 500 - obs.observe(501, Some(1)); // our batch nonce 1, block 501 - obs.observe(502, None); // not one of our batches - // Promotes the highest nonce at the block it landed in. - assert_eq!(obs.promotion(), Some((1, 501))); - } - - #[test] - fn block_observation_max_block_is_the_latest_block_with_our_batch() { - // Several of our batches across blocks; the max nonce's block wins, - // even when a still-later block has none of our batches. - let mut obs = BlockObservation::new(); - obs.observe(999, Some(0)); - obs.observe(999, Some(1)); - obs.observe(1000, Some(2)); - obs.observe(1001, None); - assert_eq!(obs.promotion(), Some((2, 1000))); - } - - #[test] - fn block_observation_with_no_observed_nonces_has_no_promotion() { - let mut obs = BlockObservation::new(); - obs.observe(500, None); - obs.observe(500, None); - obs.observe(501, None); - assert_eq!(obs.promotion(), None); - } - - #[test] - fn run_gc_drops_unreferenced_rows_and_their_filesystem_prefixes() { - let (mut storage, _db) = temp_storage_with_closed_batches("run-gc-removes-fs", 3); - let dumps_dir = tempfile::tempdir().unwrap(); - let mut app = RecordingDumpApp::new(); - - // Create two snapshots and promote both — the first becomes - // unreferenced when the second supersedes it as finalized. - take_dump_at_batch_close(&mut app, &mut storage, dumps_dir.path(), 0).unwrap(); - take_dump_at_batch_close(&mut app, &mut storage, dumps_dir.path(), 1).unwrap(); - storage.promote_finalized(0, 500).unwrap(); - storage.promote_finalized(1, 501).unwrap(); - - // The first dump's directory is on disk (RecordingDumpApp - // wrote a "state" file in it). After GC it should be gone. - let recorded = app.recorded(); - let superseded_prefix = recorded.first().expect("a dump was recorded").clone(); - assert!(superseded_prefix.exists(), "pre-GC sanity"); - - let removed = super::run_gc(&mut storage).unwrap(); - assert_eq!(removed, 1, "exactly one unreferenced dump cleaned"); - assert!(!superseded_prefix.exists(), "filesystem prefix removed too",); - - // The current finalized's prefix survives. - let finalized = storage.finalized_dump().unwrap().unwrap(); - assert!(finalized.dump.prefix.exists()); - } - - #[test] - fn run_gc_with_no_eligible_rows_is_a_noop() { - let (mut storage, _db) = temp_storage_with_closed_batches("run-gc-noop", 1); - let dumps_dir = tempfile::tempdir().unwrap(); - let mut app = RecordingDumpApp::new(); - take_dump_at_batch_close(&mut app, &mut storage, dumps_dir.path(), 0).unwrap(); - - // Pending row references the dump; nothing eligible. - let removed = super::run_gc(&mut storage).unwrap(); - assert_eq!(removed, 0); - assert!(app.recorded()[0].exists()); - } - - /// Application whose `create_dump` always fails — used to exercise - /// the atomic-close failure path. - #[derive(Default)] - struct FailingDumpApp { - progress: ApplicationProgress, - } - - impl Application for FailingDumpApp { - fn max_method_payload_bytes() -> usize { - 0 - } - - fn validate_user_op( - &self, - _sender: Address, - _user_op: &UserOp, - _current_fee: u16, - ) -> Result { - Ok(ValidationOutcome::Accept) - } - - fn apply_valid_user_op( - &mut self, - _user_op: &ValidUserOp, - safe_block: u64, - ) -> Result { - self.progress.advance(safe_block); - Ok(Vec::new()) - } - - fn apply_direct_input( - &mut self, - _input: &sequencer_core::l2_tx::DirectInput, - ) -> Result { - unimplemented!("not used in these tests") - } - - fn progress(&self) -> ApplicationProgress { - self.progress - } - - fn from_dump(_prefix: &Path) -> Result { - Ok(Self::default()) - } - - fn create_dump(&mut self, _prefix: &Path) -> Result<(), AppError> { - Err(AppError::Internal { - reason: "simulated create_dump failure".to_string(), - }) - } - - fn state_file_in_dump(prefix: &Path) -> PathBuf { - prefix.join("state") - } - } - - #[test] - fn close_batch_with_snapshot_create_dump_failure_leaves_batch_open() { - let db = temp_db("close-snapshot-create-fail"); - let mut storage = Storage::open(db.path.as_str()).expect("open"); - let mut head = storage - .initialize_open_state(0, crate::storage::SafeInputRange::empty_at(0)) - .expect("init open state"); - let open_before = head.batch_index; - - let dumps_dir = tempfile::tempdir().unwrap(); - let mut app = FailingDumpApp::default(); - let err = super::close_batch_with_snapshot( - &mut app, - &mut storage, - &mut head, - 0, - dumps_dir.path(), - ) - .expect_err("create_dump failure must abort the close"); - assert!(matches!( - err, - super::TakeDumpError::CreateDump(dump_info::CreateDumpDirError::App(_)) - )); - - // Nothing sealed: the batch is still the open Tip, no successor, - // and no pending snapshot row was written. - let open_after = storage - .open_state() - .unwrap() - .expect("tip must still be open"); - assert_eq!( - open_after.batch_index, open_before, - "batch must remain the open Tip after a create_dump failure" - ); - assert!( - storage.latest_pending_dump().unwrap().is_none(), - "no pending snapshot row when create_dump failed" - ); - } - - #[test] - fn close_batch_with_snapshot_seals_and_registers_pending_atomically() { - let db = temp_db("close-snapshot-atomic"); - let mut storage = Storage::open(db.path.as_str()).expect("open"); - let mut head = storage - .initialize_open_state(0, crate::storage::SafeInputRange::empty_at(0)) - .expect("init open state"); - let sealed_index = head.batch_index; - - let dumps_dir = tempfile::tempdir().unwrap(); - let mut app = RecordingDumpApp::new(); - super::close_batch_with_snapshot(&mut app, &mut storage, &mut head, 0, dumps_dir.path()) - .expect("atomic close"); - - // Head advanced to the freshly opened batch; the sealed batch - // has its pending snapshot row and the dump exists on disk. - assert_ne!( - head.batch_index, sealed_index, - "head should advance to the next batch" - ); - let pending = storage - .latest_pending_dump() - .unwrap() - .expect("pending row exists"); - assert_eq!( - pending.nonce, 0, - "pending keyed by the sealed batch's nonce" - ); - assert_eq!(app.recorded().len(), 1, "dump created on disk"); - assert!(pending.dump.prefix.join("state").exists()); - } -} diff --git a/sequencer/src/ingress/inclusion_lane/tests.rs b/sequencer/src/ingress/inclusion_lane/tests.rs index f75b5824..b62ef352 100644 --- a/sequencer/src/ingress/inclusion_lane/tests.rs +++ b/sequencer/src/ingress/inclusion_lane/tests.rs @@ -329,13 +329,20 @@ impl Application for ReplayRecordingApp { self.progress } - fn from_dump(_prefix: &Path) -> Result { - Ok(Self::default()) + fn from_dump(prefix: &Path) -> Result { + let bytes = std::fs::read(Self::state_file_in_dump(prefix))?; + Ok(Self { + progress: decode_progress(&bytes, "ReplayRecordingApp")?, + replayed: Vec::new(), + }) } fn create_dump(&mut self, prefix: &Path) -> Result<(), AppError> { std::fs::create_dir(prefix)?; - std::fs::write(Self::state_file_in_dump(prefix), b"")?; + std::fs::write( + Self::state_file_in_dump(prefix), + encode_progress(self.progress), + )?; Ok(()) } @@ -346,7 +353,6 @@ impl Application for ReplayRecordingApp { fn default_test_config() -> InclusionLaneConfig { InclusionLaneConfig { - batch_submitter_address: Address::from_slice(&[0xff; 20]), // A leaked tempdir per call: the lane unconditionally writes // dump artifacts there, and the test stubs' `create_dump` // creates the directory. Tempdir gets reaped by the OS. @@ -380,13 +386,11 @@ fn register_genesis_snapshot(app: &mut A, storage: &mut Storage, &super::dump_info::DumpInfo { format_version: super::dump_info::FORMAT_VERSION, next_batch_nonce: 0, - l2_tx_index: 0, - promoted_inclusion_block: Some(0), }, ) .expect("create genesis dump"); storage - .insert_finalized_dump(&dump_dir, 0, 0) + .insert_baseline_snapshot(&dump_dir, ExecutedInputCount::ZERO) .expect("insert finalized snapshot"); } @@ -399,7 +403,7 @@ async fn start_lane( tokio::task::JoinHandle>, ) { let mut storage = Storage::open(db_path).expect("open storage"); - pin_test_deployment_identity(&mut storage, config.batch_submitter_address); + pin_test_deployment_identity(&mut storage, Address::repeat_byte(0xff)); storage .append_safe_inputs(0, &[], SENDER_A, &default_protocol_timing()) .expect("seed observed safe head"); @@ -529,7 +533,8 @@ async fn sustained_rejected_queue_cannot_starve_poisoned_frontier() { storage, config, }; - let mut lane_handle = tokio::task::spawn_blocking(move || lane.run_forever(0)); + let mut lane_handle = + tokio::task::spawn_blocking(move || lane.run_forever(ExecutedInputCount::ZERO)); let producer_tx = tx.clone(); let producer = tokio::spawn(async move { @@ -590,7 +595,7 @@ fn reconciliation_digests_an_epoch_sized_outage_backlog_in_one_turn() { let db = temp_db("digestibility-epoch-backlog"); let mut storage = Storage::open(db.path.as_str()).expect("open storage"); let config = default_test_config(); - pin_test_deployment_identity(&mut storage, config.batch_submitter_address); + pin_test_deployment_identity(&mut storage, Address::repeat_byte(0xff)); storage .append_safe_inputs(0, &[], SENDER_A, &default_protocol_timing()) .expect("seed observed safe head"); @@ -651,7 +656,7 @@ fn frame_clock_waits_five_blocks_and_collapses_observation_jumps() { let db = temp_db("frame-clock-block-interval"); let mut storage = Storage::open(db.path.as_str()).expect("open storage"); let config = default_test_config(); - pin_test_deployment_identity(&mut storage, config.batch_submitter_address); + pin_test_deployment_identity(&mut storage, Address::repeat_byte(0xff)); storage .append_safe_inputs(0, &[], SENDER_A, &default_protocol_timing()) .expect("seed observed safe head"); @@ -703,21 +708,22 @@ fn frame_clock_waits_five_blocks_and_collapses_observation_jumps() { TurnOutcome::Processed ); assert!(matches!(response.try_recv(), Ok(Ok(())))); - let sequenced = lane - .storage - .ordered_l2_txs_page_from(0, 10) - .expect("read sequenced clock values"); + let sequenced = history_rows(&mut lane.storage, 10); assert_eq!(sequenced.len(), 2); assert!(matches!( - &sequenced[0].tx, - SequencedL2Tx::Direct(DirectInput { - block_number: 4, + &sequenced[0].context, + crate::storage::L2TxContext::DirectInput { + tx: DirectInput { + block_number: 4, + .. + }, .. - }) + } + )); + assert!(matches!( + &sequenced[1].context, + crate::storage::L2TxContext::UserOp { safe_block: 5, .. } )); - assert_eq!(sequenced[0].frame_safe_block, 5); - assert!(matches!(&sequenced[1].tx, SequencedL2Tx::UserOp(_))); - assert_eq!(sequenced[1].frame_safe_block, 5); lane.storage .append_safe_inputs(32, &[], SENDER_A, &default_protocol_timing()) @@ -866,7 +872,6 @@ fn seed_replay_fixture(db_path: &str) -> Vec { safe_input_index: 0, executed_input_offset: ExecutedInputCount::new(2), }], - None, ) .expect("close first frame with direct attribution"); @@ -895,7 +900,6 @@ fn seed_replay_fixture(db_path: &str) -> Vec { safe_input_index: 1, executed_input_offset: ExecutedInputCount::new(4), }], - None, ) .expect("close second frame with direct attribution"); @@ -920,7 +924,6 @@ fn seed_replay_fixture(db_path: &str) -> Vec { safe_input_index: 2, executed_input_offset: ExecutedInputCount::new(5), }], - None, ) .expect("close third frame with direct attribution"); @@ -965,7 +968,7 @@ fn read_count(db_path: &str, table: &str) -> i64 { fn read_frame_direct_count(db_path: &str, batch_index: i64, frame_in_batch: i64) -> i64 { let conn = Storage::open_connection(db_path).expect("open sqlite reader"); conn.query_row( - "SELECT COUNT(*) FROM sequenced_l2_txs + "SELECT COUNT(*) FROM application_inputs WHERE batch_index = ?1 AND frame_in_batch = ?2 AND safe_input_index IS NOT NULL", @@ -1074,10 +1077,7 @@ async fn sequenced_safe_inputs_are_drained_but_not_executed() { storage .append_safe_inputs(0, &[], SENDER_A, &default_protocol_timing()) .expect("seed observed safe head"); - let config = InclusionLaneConfig { - batch_submitter_address, - ..default_test_config() - }; + let config = default_test_config(); pin_test_deployment_identity(&mut storage, batch_submitter_address); { let mut app = SharedCountingApp::new(); @@ -1116,36 +1116,19 @@ async fn sequenced_safe_inputs_are_drained_but_not_executed() { .expect("append safe batch-submitter input"); let drained = wait_until(Duration::from_secs(2), || { - read_frame_direct_count(db.path.as_str(), 0, 1) == 1 + read_frame_safe_blocks(db.path.as_str()).last() == Some(&10) }) .await; shutdown_lane(&shutdown, lane_handle).await; - assert!( drained, - "expected sequenced safe input to be drained into frame 1" + "the frame must account for the observed L1 interval" ); - - let mut storage = Storage::open(db.path.as_str()).expect("open storage"); - let replay = storage - .ordered_l2_txs_page_from(0, 16) - .expect("load own-batch replay row"); - assert_eq!(replay.len(), 1); - assert!(matches!( - &replay[0].tx, - SequencedL2Tx::Direct(DirectInput { sender, .. }) - if *sender == batch_submitter_address - )); - assert_eq!(replay[0].executed_input_offset, None); - - // The lane's own batch input was drained into a frame and - // sequenced into `sequenced_l2_txs`, but the lane skipped - // shared application-execution boundary for it. Catch-up replays the same - // sequenced stream and also filters batch-submitter rows — so a - // fresh `SharedCountingApp` driven through `catch_up_application` - // ends with counter == 0, confirming the symmetric skip. + let mut storage = Storage::open(db.path.as_str()).unwrap(); + assert!(history_rows(&mut storage, 16).is_empty()); + assert_eq!(read_frame_direct_count(db.path.as_str(), 0, 1), 0); let mut fresh_app = SharedCountingApp::new(); - catch_up_application_paged(&mut fresh_app, &mut storage, batch_submitter_address, 0, 16) + catch_up_application_paged(&mut fresh_app, &mut storage, ExecutedInputCount::ZERO, 16) .expect("catch up"); assert_eq!( fresh_app.executed_input_count().get(), @@ -1222,11 +1205,9 @@ async fn safe_inputs_already_available_are_sequenced_before_later_user_ops() { let replay: Vec = { let mut storage = Storage::open(db.path.as_str()).expect("open storage"); - storage - .ordered_l2_txs_page_from(0, 1_000_000) - .expect("load ordered replay") + history_rows(&mut storage, 1_000_000) .into_iter() - .map(|row| row.tx) + .map(|row| context_tx(row.context)) .collect() }; shutdown_lane(&shutdown, lane_handle).await; @@ -1406,19 +1387,14 @@ fn catch_up_replays_multiple_pages() { let db = temp_db("catch-up-multi-page"); let expected = seed_replay_fixture(db.path.as_str()); let mut storage = Storage::open(db.path.as_str()).expect("open storage"); - let mappings: Vec<_> = storage - .ordered_l2_txs_page_from(0, 16) - .expect("load attributed replay rows") + let mappings: Vec<_> = history_rows(&mut storage, 16) .into_iter() - .map(|row| row.executed_input_offset.map(ExecutedInputCount::get)) + .map(|row| row.offset.get()) .collect(); - assert_eq!( - mappings, - vec![Some(0), Some(1), Some(2), Some(3), Some(4), Some(5)] - ); + assert_eq!(mappings, vec![0, 1, 2, 3, 4, 5]); let mut app = ReplayRecordingApp::default(); - catch_up_application_paged(&mut app, &mut storage, Address::from([0xff; 20]), 0, 2) + catch_up_application_paged(&mut app, &mut storage, ExecutedInputCount::ZERO, 2) .expect("catch up in pages"); assert_eq!(app.replayed, expected); @@ -1431,56 +1407,35 @@ fn catch_up_replays_multiple_pages() { } #[test] -fn catch_up_rejects_missing_mapping_before_execution() { - let db = temp_db("catch-up-missing-mapping"); - let mut storage = Storage::open(db.path.as_str()).expect("open storage"); - pin_test_deployment_identity(&mut storage, Address::from([0xff; 20])); - let mut head = storage - .initialize_open_state(0, SafeInputRange::empty_at(0)) - .expect("initialize open state"); - let (unmapped, _response) = make_pending_user_op(0x51); - storage - .append_user_ops_chunk(&mut head, &[unmapped]) - .expect("seed intentionally unmapped physical user op"); - +fn catch_up_rejects_history_gap_before_execution() { + let db = temp_db("catch-up-gap"); + seed_replay_fixture(&db.path); + let mut storage = Storage::open(&db.path).unwrap(); + let conn = Storage::open_connection(&db.path).unwrap(); + conn.execute_batch("DROP TRIGGER trg_protect_valid_application_input_delete; DELETE FROM application_inputs WHERE offset=1;").unwrap(); let mut app = ReplayRecordingApp::default(); - let err = catch_up_application_paged(&mut app, &mut storage, Address::from([0xff; 20]), 0, 2) - .expect_err("missing canonical mapping must stop catch-up"); - - assert!(matches!( - &err, - CatchUpError::ExecutionOffsetMismatch { - db_offset: 1, - kind: "user op", - expected: Some(0), - stored: None, - } - )); - assert!( - app.replayed.is_empty(), - "mapping is checked before execution" - ); - assert_eq!(app.executed_input_count(), ExecutedInputCount::ZERO); - assert!(InclusionLaneError::CatchUp { source: err }.is_terminal_invariant()); + let failure = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + let _ = catch_up_application_paged(&mut app, &mut storage, ExecutedInputCount::ZERO, 2); + })); + assert!(failure.is_err()); + assert!(app.replayed.is_empty()); } #[test] -fn catch_up_rejects_wrong_mapping_before_execution() { +fn catch_up_rejects_wrong_snapshot_count_before_execution() { let db = temp_db("catch-up-wrong-mapping"); seed_replay_fixture(db.path.as_str()); let mut storage = Storage::open(db.path.as_str()).expect("open storage"); let mut app = ReplayRecordingApp::with_executed_input_count(3); - let err = catch_up_application_paged(&mut app, &mut storage, Address::from([0xff; 20]), 0, 2) + let err = catch_up_application_paged(&mut app, &mut storage, ExecutedInputCount::ZERO, 2) .expect_err("mapping from a different application boundary must stop catch-up"); assert!(matches!( &err, - CatchUpError::ExecutionOffsetMismatch { - db_offset: 1, - kind: "user op", - expected: Some(3), - stored: Some(0), + CatchUpError::SnapshotExecutionCountMismatch { + application: 3, + storage: 0 } )); assert!( @@ -1498,7 +1453,7 @@ fn catch_up_handles_mixed_user_ops_and_direct_inputs_across_page_boundary() { let mut storage = Storage::open(db.path.as_str()).expect("open storage"); let mut app = ReplayRecordingApp::default(); - catch_up_application_paged(&mut app, &mut storage, Address::from([0xff; 20]), 0, 4) + catch_up_application_paged(&mut app, &mut storage, ExecutedInputCount::ZERO, 4) .expect("catch up across page boundary"); assert_eq!(app.replayed, expected); @@ -1583,14 +1538,12 @@ fn standard_recovery_rebases_history_and_restart_on_surviving_checkpoint() { safe_input_index: 1, executed_input_offset: direct_receipt.offset, }], - Some((0, 10)), ) .expect("drain direct and promote prefix snapshot"); let finalized_before = storage .finalized_dump() .expect("read finalized prefix") .expect("prefix snapshot promoted"); - assert_eq!(finalized_before.l2_tx_index, 1); assert_eq!( finalized_before.executed_input_count, ExecutedInputCount::new(1) @@ -1628,20 +1581,14 @@ fn standard_recovery_rebases_history_and_restart_on_surviving_checkpoint() { ExecutedInputCount::new(3) ); - let before_recovery = storage - .ordered_l2_txs_page_from(0, 32) - .expect("read pre-recovery history"); - let old_direct_physical = before_recovery - .iter() - .find_map(|row| match &row.tx { - SequencedL2Tx::Direct(value) if value.sender == direct_sender => { - assert_eq!(row.executed_input_offset, Some(ExecutedInputCount::new(1))); - Some(row.db_offset) - } - _ => None, - }) - .expect("doomed direct row exists before recovery"); - + let before_recovery = history_rows(&mut storage, 32); + assert_eq!( + before_recovery + .iter() + .map(|row| row.offset.get()) + .collect::>(), + vec![0, 1, 2] + ); let invalidated = storage .recover_post_flush_for_recovery(10, &protocol, crate::clock::unix_now_ms()) .expect("standard recovery cascade"); @@ -1662,34 +1609,19 @@ fn standard_recovery_rebases_history_and_restart_on_surviving_checkpoint() { finalized_before, "the accepted prefix checkpoint must survive the cascade" ); - assert!( - storage.latest_pending_dump().unwrap().is_none(), - "the doomed suffix checkpoint must not survive" - ); - - let after_recovery = storage - .ordered_l2_txs_page_from(0, 32) - .expect("read recovered history"); - let (new_direct_physical, new_direct_logical) = after_recovery - .iter() - .find_map(|row| match &row.tx { - SequencedL2Tx::Direct(value) if value.sender == direct_sender => { - Some((row.db_offset, row.executed_input_offset)) - } - _ => None, - }) - .expect("re-drained direct exists after recovery"); - assert!( - new_direct_physical > old_direct_physical, - "recovery must physically re-drain the invalidated direct" + assert_eq!( + storage.latest_snapshot().unwrap().unwrap().dump, + finalized_before.dump ); - assert_eq!(new_direct_logical, Some(ExecutedInputCount::new(1))); + let after_recovery = history_rows(&mut storage, 32); + assert_eq!(after_recovery.len(), 2); + assert!(matches!( + &after_recovery[1].context, + crate::storage::L2TxContext::DirectInput { input_index: 1, .. } + )); + assert_eq!(after_recovery[1].offset, ExecutedInputCount::new(1)); - // A restart loads the surviving count-1 checkpoint and catches up through - // the replacement recovery Tip. The invalidated rows are physical audit - // history only and cannot perturb application progress. let checkpoint = catch_up_snapshot(&mut storage).expect("select surviving checkpoint"); - assert_eq!(checkpoint.l2_tx_index, finalized_before.l2_tx_index); assert_eq!( checkpoint.executed_input_count, finalized_before.executed_input_count @@ -1701,8 +1633,7 @@ fn standard_recovery_rebases_history_and_restart_on_surviving_checkpoint() { catch_up_application_paged( &mut restarted, &mut storage, - batch_submitter, - checkpoint.l2_tx_index, + checkpoint.executed_input_count, 2, ) .expect("catch up through recovery re-drain"); @@ -1733,19 +1664,14 @@ fn standard_recovery_rebases_history_and_restart_on_surviving_checkpoint() { ExecutedInputCount::new(3) ); - let valid = storage - .ordered_l2_txs_page_from(0, 32) - .expect("read replacement history"); - let mappings: Vec<_> = valid - .iter() - .map(|row| row.executed_input_offset.map(ExecutedInputCount::get)) - .collect(); - assert_eq!(mappings, vec![Some(0), None, Some(1), Some(2)]); + let valid = history_rows(&mut storage, 32); + let mappings: Vec<_> = valid.iter().map(|row| row.offset.get()).collect(); + assert_eq!(mappings, vec![0, 1, 2]); let user_seeds: Vec<_> = valid .iter() - .filter_map(|row| match &row.tx { - SequencedL2Tx::UserOp(value) => Some(value.data[0]), - SequencedL2Tx::Direct(_) => None, + .filter_map(|row| match &row.context { + crate::storage::L2TxContext::UserOp { tx, .. } => Some(tx.data[0]), + crate::storage::L2TxContext::DirectInput { .. } => None, }) .collect(); assert_eq!(user_seeds, vec![0x51, 0x53]); @@ -1756,8 +1682,7 @@ fn standard_recovery_rebases_history_and_restart_on_surviving_checkpoint() { catch_up_application_paged( &mut restarted_again, &mut storage, - batch_submitter, - checkpoint.l2_tx_index, + checkpoint.executed_input_count, 2, ) .expect("catch up through replacement suffix"); @@ -1774,7 +1699,7 @@ fn catch_up_load_error_reports_offset() { let mut storage = Storage::open_writer(db.path.as_str()).expect("open raw storage"); let mut app = ReplayRecordingApp::default(); - let err = catch_up_application_paged(&mut app, &mut storage, Address::from([0xff; 20]), 0, 2) + let err = catch_up_application_paged(&mut app, &mut storage, ExecutedInputCount::ZERO, 2) .expect_err("catch up should fail without schema"); assert!(matches!(err, CatchUpError::LoadReplay { offset: 0, .. })); @@ -1785,12 +1710,12 @@ async fn lane_refuses_snapshot_whose_application_count_disagrees_with_storage() let db = temp_db("snapshot-execution-count-mismatch"); let config = default_test_config(); let mut storage = Storage::open(db.path.as_str()).expect("open storage"); - pin_test_deployment_identity(&mut storage, config.batch_submitter_address); + pin_test_deployment_identity(&mut storage, Address::repeat_byte(0xff)); storage .append_safe_inputs( 0, &[], - config.batch_submitter_address, + Address::repeat_byte(0xff), &default_protocol_timing(), ) .expect("seed observed safe head"); @@ -1962,7 +1887,7 @@ async fn restart_resumes_from_pending_checkpoint_without_skipping_txs() { let snapshotted = wait_until(Duration::from_secs(2), || { let mut s = Storage::open(db.path.as_str()).expect("open"); - s.latest_pending_dump() + s.latest_snapshot() .expect("read pending") .map(|p| read_dump_counter(&p.dump.prefix) == 3) .unwrap_or(false) @@ -1978,7 +1903,8 @@ async fn restart_resumes_from_pending_checkpoint_without_skipping_txs() { s.finalized_dump() .expect("finalized") .expect("genesis finalized exists") - .l2_tx_index, + .executed_input_count + .get(), 0, "finalized must still be genesis (nothing was promoted)" ); @@ -2008,7 +1934,7 @@ async fn restart_resumes_from_pending_checkpoint_without_skipping_txs() { let reached_four = wait_until(Duration::from_secs(2), || { let mut s = Storage::open(db.path.as_str()).expect("open"); - s.latest_pending_dump() + s.latest_snapshot() .expect("read pending") .map(|p| read_dump_counter(&p.dump.prefix) == 4) .unwrap_or(false) @@ -2016,7 +1942,7 @@ async fn restart_resumes_from_pending_checkpoint_without_skipping_txs() { .await; let observed = { let mut s = Storage::open(db.path.as_str()).expect("open"); - s.latest_pending_dump() + s.latest_snapshot() .expect("read pending") .map(|p| read_dump_counter(&p.dump.prefix)) }; @@ -2029,208 +1955,45 @@ async fn restart_resumes_from_pending_checkpoint_without_skipping_txs() { ); } -/// Regression for the empty-batch snapshot offset: a batch that closes -/// with no sequenced txs of its own still reflects state through the -/// prior global replay head. Recording `0` means a later promotion to -/// finalized would make catch-up replay the entire history again, -/// double-applying every prior tx. #[test] -fn empty_batch_snapshot_records_global_replay_head_not_genesis() { - let db = temp_db("empty-batch-snapshot-head"); - // Batch 0 gets 6 sequenced txs; close it, then close an empty batch 1. - let _expected = seed_replay_fixture(db.path.as_str()); - let mut storage = Storage::open(db.path.as_str()).expect("open storage"); - let mut head = storage - .open_state() - .expect("open state") - .expect("batch 0 is the open tip"); - storage - .close_frame_and_batch(&mut head, 30) - .expect("close batch 0 (6 txs)"); - storage - .close_frame_and_batch(&mut head, 30) - .expect("close empty batch 1"); - - let dumps_dir = tempfile::tempdir().expect("dumps dir"); - let mut app = TestApp::default(); - super::snapshot::take_dump_at_batch_close(&mut app, &mut storage, dumps_dir.path(), 1) - .expect("take dump for empty batch 1"); - - let pending = storage - .latest_pending_dump() - .expect("read pending") - .expect("pending row for batch 1"); - assert_eq!( - pending.nonce, 1, - "snapshot keyed by the empty batch's nonce" - ); - - let global_head: u64 = { - let conn = Storage::open_connection(db.path.as_str()).expect("open reader"); - conn.query_row("SELECT MAX(offset) FROM sequenced_l2_txs", [], |row| { - row.get::<_, i64>(0) - }) - .expect("max offset") as u64 - }; - - assert_eq!( - pending.l2_tx_index, global_head, - "empty-batch snapshot must record the global replay head ({global_head}), not genesis (0); \ - otherwise catch-up after this snapshot is finalized replays the whole stream and double-applies it" - ); +fn empty_batch_snapshot_preserves_application_count() { + let db = temp_db("empty-snapshot-count"); + let expected = seed_replay_fixture(&db.path); + let mut storage = Storage::open(&db.path).unwrap(); + let mut app = ReplayRecordingApp::default(); + catch_up_application_paged(&mut app, &mut storage, ExecutedInputCount::ZERO, 2).unwrap(); + assert_eq!(app.replayed, expected); + let mut head = storage.open_state().unwrap().unwrap(); + let dumps = tempfile::tempdir().unwrap(); + super::snapshot::close_batch_with_snapshot(&mut app, &mut storage, &mut head, 30, dumps.path()) + .unwrap(); + super::snapshot::close_batch_with_snapshot(&mut app, &mut storage, &mut head, 30, dumps.path()) + .unwrap(); + let snapshot = storage.latest_snapshot().unwrap().unwrap(); + assert_eq!(snapshot.executed_input_count.get(), 6); + let restored = + ReplayRecordingApp::from_dump(&super::dump_info::app_prefix(&snapshot.dump.prefix)) + .unwrap(); + assert_eq!(restored.executed_input_count().get(), 6); } -/// Regression for the promote/drain wedge. The pre-fix per-block path promoted -/// a batch in one transaction and advanced the safe-input drain -/// (`close_frame_only`) in a *separate* one. A crash between them left a -/// promoted-but-undrained batch; on restart the lane re-processed the same safe -/// input, re-derived the accepted nonce, and called `promote_finalized` on a -/// now-deleted pending row → `QueryReturnedNoRows` → crash-loop (verified -/// against the SQL: `accepted_batch_nonce_at` has no pending-row gate, and -/// `next_undrained` advances only when inputs are sequenced by the drain). -/// -/// `close_frame_only_with_executions` folds the promotion into the drain's -/// transaction, so a committed promotion always comes with an advanced drain — -/// the wedge state is unrepresentable. -#[test] -fn promotion_advances_drain_atomically_so_restart_cannot_re_promote() { - let db = temp_db("promote-drain-atomic"); - let mut storage = Storage::open(db.path.as_str()).expect("open storage"); - pin_test_deployment_identity(&mut storage, SENDER_A); - let mut head = storage - .initialize_open_state(0, SafeInputRange::empty_at(0)) - .expect("open batch 0"); - - // Our batch 0 lands on L1 as safe input 0; the scheduler accepts it as - // nonce 0 (this is what populates `safe_accepted_batches`). - let batch0 = StoredSafeInput { - sender: SENDER_A, - payload: ssz::Encode::as_ssz_bytes(&sequencer_core::batch::Batch { - nonce: 0, - frames: Vec::new(), - }), - block_number: 100, - }; - // DeferUntilAnchorSet skips acceptance simulation: this test's subject is - // promote/drain atomicity, and a hand-built landing payload cannot - // content-match an unsealed local batch — running the content-identity - // check here would record a divergence marker and (correctly) freeze the - // batch tree via the I15 triggers. - storage - .append_safe_inputs_with_timestamp( - 100, - 100, - std::slice::from_ref(&batch0), - SENDER_A, - &default_protocol_timing(), - crate::storage::FrontierMode::DeferUntilAnchorSet, - ) - .expect("append our batch as a safe input"); - - // Close batch 0 off-chain and register its pending snapshot. +fn history_rows(storage: &mut Storage, limit: usize) -> Vec { + let bounds = storage.history_bounds().unwrap(); storage - .close_frame_and_batch(&mut head, 100) - .expect("close batch 0"); - let dumps = tempfile::tempdir().expect("dumps dir"); - super::snapshot::take_dump_at_batch_close( - &mut TestApp::default(), - &mut storage, - dumps.path(), - 0, - ) - .expect("pending snapshot for batch 0"); - - // The lane advances the safe frontier over batch 0's landing: it promotes - // batch 0 AND sequences the drain in one transaction. - storage - .close_frame_only_with_executions( - &mut head, - 100, - SafeInputRange::new(0, 1), - &[], - Some((0, 100)), + .canonical_history_page( + sequencer_core::history::HistoryClaim { + version: bounds.version, + next_input: bounds.available_from, + }, + limit, ) - .expect("atomic close-frame + promote"); - - // The promotion committed... - assert_eq!( - storage - .finalized_dump() - .unwrap() - .expect("finalized") - .inclusion_block, - 100, - ); - // ...and the drain advanced past batch 0's safe input in the SAME commit, so - // a restart resumes *after* it and never re-processes (hence never - // re-promotes) the batch whose pending row promotion deleted. - assert!( - storage.next_undrained_safe_input_index().unwrap() > 0, - "drain must advance past the promoted batch's safe input atomically with \ - the promotion — otherwise a restart re-processes safe input 0 and \ - re-promotes a now-deleted pending row (QueryReturnedNoRows wedge)", - ); + .unwrap() + .rows } -/// The atomicity complement: if the promotion fails *inside* the combined -/// transaction (here, a missing pending row), the drain advance rolls back with -/// it. There is never a half-applied "drained but not promoted" state — the -/// mirror of the wedge. -#[test] -fn frame_promotion_failure_rolls_back_drain_execution_offsets_and_head() { - let db = temp_db("close-promote-rollback"); - let mut storage = Storage::open(db.path.as_str()).expect("open storage"); - pin_test_deployment_identity(&mut storage, SENDER_A); - let mut head = storage - .initialize_open_state(0, SafeInputRange::empty_at(0)) - .expect("open batch 0"); - - // A safe input exists to drain, but there is no pending snapshot for the - // nonce we ask to promote, so `promote_finalized_in` errors mid-transaction. - let direct = StoredSafeInput { - sender: Address::ZERO, - payload: vec![1], - block_number: 100, - }; - storage - .append_safe_inputs( - 100, - std::slice::from_ref(&direct), - SENDER_A, - &default_protocol_timing(), - ) - .expect("append safe input"); - - let result = storage.close_frame_only_with_executions( - &mut head, - 100, - SafeInputRange::new(0, 1), - &[DirectInputExecution { - safe_input_index: 0, - executed_input_offset: ExecutedInputCount::ZERO, - }], - Some((7, 100)), - ); - assert!( - result.is_err(), - "promoting a missing pending row must fail the whole call", - ); - - // Nothing committed: no finalized promotion, and the drain did not advance. - assert!( - storage.finalized_dump().unwrap().is_none(), - "no finalized snapshot after the failed promotion", - ); - assert_eq!( - storage.next_undrained_safe_input_index().unwrap(), - 0, - "the drain rolled back together with the failed promotion", - ); - assert_eq!( - storage.next_executed_input_count().unwrap(), - ExecutedInputCount::ZERO - ); - assert_eq!(head.frame_in_batch, 0); - assert_eq!(head.safe_block, 0); - assert_eq!(read_frame_safe_blocks(db.path.as_str()), vec![0]); +fn context_tx(context: crate::storage::L2TxContext) -> SequencedL2Tx { + match context { + crate::storage::L2TxContext::UserOp { tx, .. } => SequencedL2Tx::UserOp(tx), + crate::storage::L2TxContext::DirectInput { tx, .. } => SequencedL2Tx::Direct(tx), + } } diff --git a/sequencer/src/integration_tests/chain_id_validation.rs b/sequencer/src/integration_tests/chain_id_validation.rs index 150f84d7..324bf82f 100644 --- a/sequencer/src/integration_tests/chain_id_validation.rs +++ b/sequencer/src/integration_tests/chain_id_validation.rs @@ -95,9 +95,14 @@ fn seed_setup_complete(db_path: &str, chain_id: u64, submitter: Address) { .expect("seed deployment identity"); let snapshot_prefix = std::path::Path::new(db_path).with_file_name("seed-finalized"); storage - .insert_initial_finalized_dump(&snapshot_prefix, 0, 0, 0, 0) + .complete_baseline_setup( + &snapshot_prefix, + sequencer_core::history::ExecutedInputCount::ZERO, + 0, + 0, + false, + ) .expect("seed finalized snapshot fact"); - storage.complete_setup().expect("complete setup"); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] diff --git a/sequencer/src/integration_tests/e2e_sequencer.rs b/sequencer/src/integration_tests/e2e_sequencer.rs index 13f22f19..8b871497 100644 --- a/sequencer/src/integration_tests/e2e_sequencer.rs +++ b/sequencer/src/integration_tests/e2e_sequencer.rs @@ -4,6 +4,7 @@ use std::io::ErrorKind; use std::time::Duration; +use crate::storage::{ApplicationInputRow, L2TxContext}; use alloy_primitives::{Address, Signature, U256}; use alloy_sol_types::{Eip712Domain, SolStruct}; use app_core::application::{ @@ -21,7 +22,7 @@ use sequencer::runtime::shutdown::RuntimeScope; use sequencer::storage::{DeploymentIdentity, FeeOracleIdentity, Storage, StoredSafeInput}; use sequencer_core::api::{TxRequest, TxResponse, WsTxMessage}; use sequencer_core::application::Application; -use sequencer_core::l2_tx::SequencedL2Tx; +use sequencer_core::history::{ExecutedInputCount, HistoryClaim}; use sequencer_core::user_op::UserOp; use sequencer_rust_client::SequencerClient; use tokio::io::{AsyncReadExt, AsyncWriteExt}; @@ -210,7 +211,7 @@ async fn e2e_submit_tx_ack_and_broadcast() { let endpoint = format!("http://{}", runtime.addr); let client = SequencerClient::new_with_timeout(endpoint.clone(), Duration::from_secs(2)) .expect("build sequencer client"); - let ws_url = client.ws_subscribe_url(0); + let ws_url = client.ws_subscribe_url(history_claim(&db.path, 0)); let (mut ws, _) = tokio::time::timeout(Duration::from_secs(5), connect_async(ws_url)) .await .expect("timeout connecting websocket") @@ -219,7 +220,7 @@ async fn e2e_submit_tx_ack_and_broadcast() { // The deposit is broadcast first. let deposit_message = recv_ws_message(&mut ws).await; match deposit_message { - WsTxMessage::DirectInput { offset, .. } => assert_eq!(offset, 1), + WsTxMessage::DirectInput { offset, .. } => assert_eq!(offset, 0), other => panic!("expected deposit direct input as first WS message, got {other:?}"), } let method = Method::Withdrawal(Withdrawal { @@ -276,7 +277,7 @@ async fn e2e_submit_tx_ack_and_broadcast() { data, .. } => { - assert_eq!(offset, 2); + assert_eq!(offset, 1); assert_eq!(ws_sender, sender.to_string()); // Frame fee includes the default tenfold log-space slack. assert_eq!(fee, 1356); @@ -1026,7 +1027,7 @@ async fn restart_replays_same_ordered_l2_tx_stream_from_db() { let endpoint = format!("http://{}", runtime.addr); let client = SequencerClient::new_with_timeout(endpoint.clone(), Duration::from_secs(2)) .expect("build sequencer client"); - let ws_url = client.ws_subscribe_url(0); + let ws_url = client.ws_subscribe_url(history_claim(&db.path, 0)); let (mut ws, _) = tokio::time::timeout(Duration::from_secs(5), connect_async(ws_url)) .await .expect("timeout connecting websocket") @@ -1056,10 +1057,10 @@ async fn restart_replays_same_ordered_l2_tx_stream_from_db() { 3, "expected deposit, direct input, and user op" ); - // DB offsets (SQLite rowid) start at 1. - assert_ws_message_matches_tx(deposit_live, &expected[0], 1); - assert_ws_message_matches_tx(first_live, &expected[1], 2); - assert_ws_message_matches_tx(second_live, &expected[2], 3); + // Application offsets start at zero. + assert_ws_message_matches_tx(deposit_live, &expected[0], 0); + assert_ws_message_matches_tx(first_live, &expected[1], 1); + assert_ws_message_matches_tx(second_live, &expected[2], 2); shutdown_runtime(runtime).await; @@ -1071,7 +1072,7 @@ async fn restart_replays_same_ordered_l2_tx_stream_from_db() { let restarted_client = SequencerClient::new_with_timeout(restarted_endpoint, Duration::from_secs(2)) .expect("build sequencer client after restart"); - let restarted_ws_url = restarted_client.ws_subscribe_url(0); + let restarted_ws_url = restarted_client.ws_subscribe_url(history_claim(&db.path, 0)); let (mut restarted_ws, _) = tokio::time::timeout(Duration::from_secs(5), connect_async(restarted_ws_url)) .await @@ -1080,8 +1081,8 @@ async fn restart_replays_same_ordered_l2_tx_stream_from_db() { for (i, expected_tx) in expected.iter().enumerate() { let replayed = recv_ws_message(&mut restarted_ws).await; - // DB offsets start at 1. - assert_ws_message_matches_tx(replayed, expected_tx, (i + 1) as u64); + // Replaying preserves canonical offsets. + assert_ws_message_matches_tx(replayed, expected_tx, i as u64); } drop(restarted_ws); @@ -1148,13 +1149,11 @@ async fn start_full_server_with_max_body( &dump_info::DumpInfo { format_version: dump_info::FORMAT_VERSION, next_batch_nonce: 0, - l2_tx_index: 0, - promoted_inclusion_block: Some(0), }, ) .expect("genesis dump"); storage - .insert_finalized_dump(&genesis_dir, 0, 0) + .insert_baseline_snapshot(&genesis_dir, ExecutedInputCount::ZERO) .expect("register genesis"); } @@ -1175,7 +1174,6 @@ async fn start_full_server_with_max_body( shutdown.clone(), storage, InclusionLaneConfig { - batch_submitter_address: TEST_BATCH_SUBMITTER, dumps_dir, max_user_ops_per_chunk: 32, safe_input_buffer_capacity: 32, @@ -1193,7 +1191,6 @@ async fn start_full_server_with_max_body( page_size: 64, // Sentinel submitter: the WS assertions here observe the // unfiltered stream, so pass an address no fixture seeds. - ..L2TxFeedConfig::new(alloy_primitives::Address::repeat_byte(0x7f)) }, ); @@ -1250,7 +1247,6 @@ async fn start_api_only_server( page_size: 64, // Sentinel submitter: the WS assertions here observe the // unfiltered stream, so pass an address no fixture seeds. - ..L2TxFeedConfig::new(alloy_primitives::Address::repeat_byte(0x7f)) }, ); let server_task = http::start_on_listener( @@ -1390,7 +1386,7 @@ fn seed_safe_direct_input(db_path: &str, safe_block: u64, payload: Vec) { payload, block_number: safe_block, }], - Address::ZERO, + TEST_BATCH_SUBMITTER, &sequencer_core::protocol::ProtocolTiming { max_wait_blocks: sequencer_core::MAX_WAIT_BLOCKS, preemptive_margin_blocks: 75, @@ -1401,22 +1397,20 @@ fn seed_safe_direct_input(db_path: &str, safe_block: u64, payload: Vec) { .expect("append safe direct input"); } -fn all_ordered_l2_txs(db_path: &str) -> Vec { +fn all_ordered_l2_txs(db_path: &str) -> Vec { let mut storage = Storage::open_read_only(db_path).expect("open read-only storage"); storage - .ordered_l2_txs_page_from(0, 1_000_000) + .canonical_history_page(history_claim(db_path, 0), 1_000_000) .expect("load ordered l2 txs") - .into_iter() - .map(|row| row.tx) - .collect() + .rows } fn assert_ws_message_matches_tx( actual: WsTxMessage, - expected: &SequencedL2Tx, + expected: &ApplicationInputRow, expected_offset: u64, ) { - match (actual, expected) { + match (actual, &expected.context) { ( WsTxMessage::UserOp { offset, @@ -1425,7 +1419,7 @@ fn assert_ws_message_matches_tx( data, .. }, - SequencedL2Tx::UserOp(expected), + L2TxContext::UserOp { tx: expected, .. }, ) => { assert_eq!(offset, expected_offset); assert_eq!( @@ -1443,7 +1437,7 @@ fn assert_ws_message_matches_tx( payload, .. }, - SequencedL2Tx::Direct(expected), + L2TxContext::DirectInput { tx: expected, .. }, ) => { assert_eq!(offset, expected_offset); assert_eq!( @@ -1579,3 +1573,14 @@ fn decode_hex_prefixed(value: &str) -> Vec { fn test_domain() -> Eip712Domain { sequencer_core::build_input_domain(1, Address::from_slice(&[0_u8; 20])) } + +fn history_claim(db_path: &str, next: u64) -> HistoryClaim { + HistoryClaim { + version: Storage::open_read_only(db_path) + .unwrap() + .history_state() + .unwrap() + .version, + next_input: ExecutedInputCount::new(next), + } +} diff --git a/sequencer/src/integration_tests/snapshot_endpoints.rs b/sequencer/src/integration_tests/snapshot_endpoints.rs index 54aba477..11b0eb78 100644 --- a/sequencer/src/integration_tests/snapshot_endpoints.rs +++ b/sequencer/src/integration_tests/snapshot_endpoints.rs @@ -67,7 +67,7 @@ async fn start_server(db_path: &str) -> Option { let tx_feed = L2TxFeed::new( db_path.to_string(), shutdown.clone(), - L2TxFeedConfig::new(alloy_primitives::Address::repeat_byte(0x7f)), + L2TxFeedConfig::default(), ); let task = http::start_on_listener( listener, @@ -92,7 +92,7 @@ async fn start_server(db_path: &str) -> Option { /// Build a structured dump dir mirroring production layout: the app's /// state under `state`, plus a minimal `info.toml`. -fn write_state(dir: &Path, name: &str, bytes: &[u8]) -> std::path::PathBuf { +fn write_state(dir: &Path, name: &str, bytes: &[u8], next_batch_nonce: u64) -> std::path::PathBuf { let dump_dir = dir.join(name); let app_prefix = dump_info::app_prefix(&dump_dir); std::fs::create_dir_all(&app_prefix).expect("mkdir dump"); @@ -101,43 +101,83 @@ fn write_state(dir: &Path, name: &str, bytes: &[u8]) -> std::path::PathBuf { &dump_dir, &dump_info::DumpInfo { format_version: dump_info::FORMAT_VERSION, - next_batch_nonce: 0, - l2_tx_index: 0, - promoted_inclusion_block: None, + next_batch_nonce, }, ) .expect("write info.toml"); dump_dir } -fn register_finalized( +fn register_accepted( db_path: &str, dir: &Path, name: &str, bytes: &[u8], inclusion_block: u64, - l2_tx_index: u64, ) -> i64 { - let prefix = write_state(dir, name, bytes); - Storage::open(db_path) - .expect("open") - .insert_finalized_dump(&prefix, inclusion_block, l2_tx_index) - .expect("register finalized") + let mut storage = Storage::open(db_path).unwrap(); + let sender = alloy_primitives::Address::repeat_byte(0x7f); + crate::storage::test_helpers::pin_test_deployment_identity(&mut storage, sender); + let mut head = storage.open_state().unwrap().unwrap_or_else(|| { + storage + .initialize_open_state(inclusion_block, crate::storage::SafeInputRange::empty_at(0)) + .unwrap() + }); + let nonce = storage.batch_nonce(head.batch_index).unwrap(); + let prefix = write_state(dir, name, bytes, nonce + 1); + let index = head.batch_index; + let safe_block = head.safe_block; + storage + .close_frame_and_batch_with_snapshot( + &mut head, + safe_block, + &prefix, + index, + crate::storage::ExecutedInputCount::ZERO, + ) + .unwrap(); + crate::storage::test_helpers::seed_safe_inputs_with_batch_nonces( + &mut storage, + sender, + inclusion_block, + &[nonce], + ); + storage.finalized_dump().unwrap().unwrap().dump.id } -fn register_pending( - db_path: &str, - dir: &Path, - name: &str, - bytes: &[u8], - nonce: u64, - l2_tx_index: u64, -) -> i64 { - let prefix = write_state(dir, name, bytes); - Storage::open(db_path) - .expect("open") - .insert_pending_dump(&prefix, nonce, l2_tx_index) - .expect("register pending") +fn register_optimistic(db_path: &str, dir: &Path, name: &str, bytes: &[u8], nonce: u64) -> i64 { + let mut storage = Storage::open(db_path).unwrap(); + let mut head = storage.open_state().unwrap().unwrap(); + while storage.batch_nonce(head.batch_index).unwrap() <= nonce { + let current = storage.batch_nonce(head.batch_index).unwrap(); + let dump_name = if current == nonce { + name.to_owned() + } else { + format!("intermediate-{current}") + }; + let prefix = write_state(dir, &dump_name, bytes, current + 1); + let index = head.batch_index; + let safe_block = head.safe_block; + storage + .close_frame_and_batch_with_snapshot( + &mut head, + safe_block, + &prefix, + index, + crate::storage::ExecutedInputCount::ZERO, + ) + .unwrap(); + } + storage.latest_snapshot().unwrap().unwrap().dump.id +} + +fn archive_state(bytes: &[u8]) -> Vec { + let dir = tempfile::tempdir().unwrap(); + tar::Archive::new(bytes).unpack(dir.path()).unwrap(); + std::fs::read(WalletApp::state_file_in_dump(&dump_info::app_prefix( + dir.path(), + ))) + .unwrap() } /// Transient WAL lock contention vs. a real read failure. @@ -227,7 +267,7 @@ async fn finalized_state_round_trips_bytes_and_headers() { let db = temp_db("snap-finalized-roundtrip"); let dir = tempfile::tempdir().expect("dumps dir"); let state = b"the-canonical-finalized-state".to_vec(); - let dump_id = register_finalized(db.path.as_str(), dir.path(), "fin", &state, 4242, 7); + let dump_id = register_accepted(db.path.as_str(), dir.path(), "fin", &state, 4242); let Some(server) = start_server(db.path.as_str()).await else { return; }; @@ -240,7 +280,9 @@ async fn finalized_state_round_trips_bytes_and_headers() { assert_eq!(resp.status().as_u16(), 200); assert_eq!(resp.headers()["content-type"], "application/octet-stream"); assert_eq!(resp.headers()["x-inclusion-block"], "4242"); - assert_eq!(resp.headers()["x-l2-tx-index"], "7"); + assert_eq!(resp.headers()["x-executed-input-count"], "0"); + assert!(resp.headers().contains_key("x-history-era")); + assert_eq!(resp.headers()["x-recovery-generation"], "0"); assert_eq!(resp.headers()["etag"], "\"block-4242\""); let body = resp.bytes().await.expect("body"); assert_eq!( @@ -259,7 +301,7 @@ async fn finalized_state_round_trips_bytes_and_headers() { async fn finalized_inclusion_block_returns_cheap_json() { let db = temp_db("snap-inclusion-block"); let dir = tempfile::tempdir().expect("dumps dir"); - register_finalized(db.path.as_str(), dir.path(), "fin", b"x", 999, 42); + register_accepted(db.path.as_str(), dir.path(), "fin", b"x", 999); let Some(server) = start_server(db.path.as_str()).await else { return; }; @@ -271,14 +313,14 @@ async fn finalized_inclusion_block_returns_cheap_json() { let body = resp.bytes().await.expect("body"); let text = String::from_utf8_lossy(&body); assert!(text.contains("\"inclusion_block\":999"), "got: {text}"); - assert!(text.contains("\"l2_tx_index\":42"), "got: {text}"); + assert!(text.contains("\"executed_input_count\":0"), "got: {text}"); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn endpoints_404_when_no_finalized_snapshot() { let db = temp_db("snap-404"); // Migrate the schema (as startup does) but register no snapshot: the - // endpoints then see an empty `finalized_snapshot` and return 404. + // endpoints then see no registered snapshot and return 404. Storage::open(db.path.as_str()).expect("init schema"); let Some(server) = start_server(db.path.as_str()).await else { return; @@ -304,7 +346,7 @@ async fn endpoints_404_when_no_finalized_snapshot() { async fn finalized_state_if_none_match_returns_304() { let db = temp_db("snap-etag"); let dir = tempfile::tempdir().expect("dumps dir"); - let dump_id = register_finalized(db.path.as_str(), dir.path(), "fin", b"abc", 500, 3); + let dump_id = register_accepted(db.path.as_str(), dir.path(), "fin", b"abc", 500); let Some(server) = start_server(db.path.as_str()).await else { return; }; @@ -326,25 +368,11 @@ async fn finalized_state_if_none_match_returns_304() { } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn latest_snapshot_prefers_pending() { +async fn latest_snapshot_prefers_newest_valid_batch() { let db = temp_db("snap-latest-pending"); let dir = tempfile::tempdir().expect("dumps dir"); - register_finalized( - db.path.as_str(), - dir.path(), - "fin", - b"finalized-bytes", - 100, - 5, - ); - let pending_id = register_pending( - db.path.as_str(), - dir.path(), - "pend", - b"pending-bytes", - 3, - 11, - ); + register_accepted(db.path.as_str(), dir.path(), "fin", b"finalized-bytes", 100); + let pending_id = register_optimistic(db.path.as_str(), dir.path(), "pend", b"pending-bytes", 3); let Some(server) = start_server(db.path.as_str()).await else { return; }; @@ -353,10 +381,12 @@ async fn latest_snapshot_prefers_pending() { .await .expect("request"); assert_eq!(resp.status().as_u16(), 200); - assert_eq!(resp.headers()["x-l2-tx-index"], "11"); + assert_eq!(resp.headers()["x-executed-input-count"], "0"); + assert!(resp.headers().contains_key("x-history-era")); + assert_eq!(resp.headers()["x-recovery-generation"], "0"); let body = resp.bytes().await.expect("body"); assert_eq!( - body.as_ref(), + archive_state(body.as_ref()).as_slice(), b"pending-bytes", "serves the latest pending, not the finalized fallback" ); @@ -369,7 +399,7 @@ async fn finalized_state_streams_large_file() { let db = temp_db("snap-large"); let dir = tempfile::tempdir().expect("dumps dir"); let big = vec![0x5Au8; 8 * 1024 * 1024]; - let dump_id = register_finalized(db.path.as_str(), dir.path(), "big", &big, 1, 1); + let dump_id = register_accepted(db.path.as_str(), dir.path(), "big", &big, 1); let Some(server) = start_server(db.path.as_str()).await else { return; }; @@ -396,7 +426,7 @@ async fn finalized_state_disconnect_mid_stream_releases_lease() { // 8 MiB so the body is still streaming (not fully in the socket buffer) // when we disconnect. let big = vec![0xABu8; 8 * 1024 * 1024]; - let dump_id = register_finalized(db.path.as_str(), dir.path(), "big", &big, 7, 9); + let dump_id = register_accepted(db.path.as_str(), dir.path(), "big", &big, 7); let Some(server) = start_server(db.path.as_str()).await else { return; }; @@ -435,7 +465,7 @@ async fn finalized_state_concurrent_streams_count_and_release_independently() { let db = temp_db("snap-concurrent"); let dir = tempfile::tempdir().expect("dumps dir"); let big = vec![0xCDu8; 8 * 1024 * 1024]; - let dump_id = register_finalized(db.path.as_str(), dir.path(), "big", &big, 1, 1); + let dump_id = register_accepted(db.path.as_str(), dir.path(), "big", &big, 1); let Some(server) = start_server(db.path.as_str()).await else { return; }; @@ -482,7 +512,7 @@ async fn live_stream_blocks_gc_until_it_drops() { let db = temp_db("snap-stream-blocks-gc"); let dir = tempfile::tempdir().expect("dumps dir"); let big = vec![0xABu8; 8 * 1024 * 1024]; - let served_id = register_finalized(db.path.as_str(), dir.path(), "served", &big, 1, 1); + let served_id = register_accepted(db.path.as_str(), dir.path(), "served", &big, 1); let Some(server) = start_server(db.path.as_str()).await else { return; }; @@ -498,13 +528,8 @@ async fn live_stream_blocks_gc_until_it_drops() { stream.next().await.expect("chunk").expect("ok"); assert!(wait_for_lease(db.path.as_str(), served_id, 1).await); - // Promote a new finalized so the dump being streamed becomes unreferenced. - { - let prefix = write_state(dir.path(), "new", b"new-finalized"); - let mut storage = Storage::open(db.path.as_str()).expect("open"); - storage.insert_pending_dump(&prefix, 0, 2).expect("pending"); - storage.promote_finalized(0, 2).expect("promote"); - } + // Acceptance advances independently of the lane's L1 reconciliation. + register_accepted(db.path.as_str(), dir.path(), "new", b"new-finalized", 2); // GC must skip the superseded dump while the stream holds its lease. let removed = { @@ -540,7 +565,7 @@ async fn finalized_state_missing_registered_artifact_aborts_process() { } let db = temp_db("snap-missing-artifact"); let dir = tempfile::tempdir().expect("dumps dir"); - register_finalized(db.path.as_str(), dir.path(), "fin", b"bytes", 1, 1); + register_accepted(db.path.as_str(), dir.path(), "fin", b"bytes", 1); // The durable row promises the artifact exists. This is a terminal // invariant failure, so the process stops before HTTP or lease cleanup. std::fs::remove_file(WalletApp::state_file_in_dump(&dump_info::app_prefix( @@ -556,17 +581,10 @@ async fn finalized_state_missing_registered_artifact_aborts_process() { } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn latest_snapshot_falls_back_to_finalized_when_no_pending() { +async fn latest_snapshot_restores_accepted_batch_when_it_is_newest() { let db = temp_db("snap-latest-fallback"); let dir = tempfile::tempdir().expect("dumps dir"); - let fin_id = register_finalized( - db.path.as_str(), - dir.path(), - "fin", - b"finalized-bytes", - 100, - 5, - ); + let fin_id = register_accepted(db.path.as_str(), dir.path(), "fin", b"finalized-bytes", 100); // No pending registered. let Some(server) = start_server(db.path.as_str()).await else { return; @@ -576,9 +594,15 @@ async fn latest_snapshot_falls_back_to_finalized_when_no_pending() { .await .expect("request"); assert_eq!(resp.status().as_u16(), 200); - assert_eq!(resp.headers()["x-l2-tx-index"], "5"); + assert_eq!(resp.headers()["x-executed-input-count"], "0"); + assert!(resp.headers().contains_key("x-history-era")); + assert_eq!(resp.headers()["x-recovery-generation"], "0"); let body = resp.bytes().await.expect("body"); - assert_eq!(body.as_ref(), b"finalized-bytes", "falls back to finalized"); + assert_eq!( + archive_state(body.as_ref()).as_slice(), + b"finalized-bytes", + "restores finalized artifact" + ); assert!(wait_for_lease(db.path.as_str(), fin_id, 0).await); } @@ -672,7 +696,7 @@ async fn cors_permits_browser_preflight_on_fee() { async fn cors_is_limited_to_ingress_and_covers_rejections() { let db = temp_db("cors-route-scope"); let dumps = tempfile::tempdir().expect("snapshot directory"); - let dump_id = register_finalized(&db.path, dumps.path(), "finalized", b"canonical", 0, 0); + let dump_id = register_accepted(&db.path, dumps.path(), "finalized", b"canonical", 1); let Some(server) = start_server(db.path.as_str()).await else { return; }; @@ -715,3 +739,80 @@ async fn cors_is_limited_to_ingress_and_covers_rejections() { } assert!(wait_for_lease(&db.path, dump_id, 0).await); } + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn sdk_snapshot_restores_application_and_recovery_export_is_self_contained() { + let db = temp_db("snapshot-archive-roundtrip"); + let dir = tempfile::tempdir().unwrap(); + let app = WalletApp::default(); + let encoded = app_core::wallet_snapshot::encode(&app); + register_accepted(&db.path, dir.path(), "accepted", &encoded, 1); + let Some(server) = start_server(&db.path).await else { + return; + }; + let client = + sequencer_rust_client::SequencerClient::new(format!("http://{}", server.addr)).unwrap(); + let snapshot = client.latest_snapshot().await.unwrap(); + let claim = snapshot.claim; + assert_eq!( + snapshot.response.headers()["content-type"], + "application/x-tar" + ); + let bytes = snapshot.response.bytes().await.unwrap(); + let restored_dir = tempfile::tempdir().unwrap(); + tar::Archive::new(bytes.as_ref()) + .unpack(restored_dir.path()) + .unwrap(); + let restored = WalletApp::from_dump(&dump_info::app_prefix(restored_dir.path())).unwrap(); + assert_eq!(restored.executed_input_count(), claim.next_input); + assert!(!restored_dir.path().join("checkpoint.toml").exists()); + let mut stream = client.subscribe(claim).await.unwrap(); + stream.close(None).await.unwrap(); + + let export = reqwest::get(server.url("/finalized_snapshot")) + .await + .unwrap(); + assert_eq!(export.headers()["x-inclusion-block"], "1"); + let export_dir = tempfile::tempdir().unwrap(); + tar::Archive::new(export.bytes().await.unwrap().as_ref()) + .unpack(export_dir.path()) + .unwrap(); + let receipt = dump_info::read_checkpoint_info(export_dir.path()).unwrap(); + assert_eq!(receipt.inclusion_block, 1); + assert_eq!(receipt.next_batch_nonce, 1); + assert_eq!( + dump_info::read_info(export_dir.path()) + .unwrap() + .next_batch_nonce, + receipt.next_batch_nonce + ); + assert_eq!( + WalletApp::from_dump(&dump_info::app_prefix(export_dir.path())) + .unwrap() + .progress(), + app.progress() + ); + assert!( + !dir.path().join("accepted/checkpoint.toml").exists(), + "export must leave immutable local artifacts unchanged" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn archive_disconnect_releases_both_stream_and_producer_lease_owners() { + let db = temp_db("archive-disconnect"); + let dir = tempfile::tempdir().unwrap(); + let id = register_accepted(&db.path, dir.path(), "large", &vec![1; 8 * 1024 * 1024], 1); + let Some(server) = start_server(&db.path).await else { + return; + }; + let response = reqwest::get(server.url("/latest_snapshot")).await.unwrap(); + let mut stream = response.bytes_stream(); + stream.next().await.unwrap().unwrap(); + assert!(wait_for_lease(&db.path, id, 1).await); + drop(stream); + assert!( + wait_for_lease(&db.path, id, 0).await, + "disconnected archive producer must terminate and release its lease" + ); +} diff --git a/sequencer/src/integration_tests/ws_broadcaster.rs b/sequencer/src/integration_tests/ws_broadcaster.rs index 3d44ee9f..04d3ca99 100644 --- a/sequencer/src/integration_tests/ws_broadcaster.rs +++ b/sequencer/src/integration_tests/ws_broadcaster.rs @@ -4,17 +4,18 @@ use std::io::ErrorKind; use std::time::{Duration, SystemTime}; +use crate::storage::{ApplicationInputRow, L2TxContext}; use alloy_primitives::{Address, Signature}; use alloy_sol_types::Eip712Domain; use app_core::application::MAX_METHOD_PAYLOAD_BYTES; use futures_util::{SinkExt, StreamExt}; use sequencer::egress::l2_tx_feed::{L2TxFeed, L2TxFeedConfig}; -use sequencer::http::{self, ApiConfig, WS_CATCHUP_WINDOW_EXCEEDED_REASON}; +use sequencer::http::{self, ApiConfig}; use sequencer::ingress::inclusion_lane::{PendingUserOp, SequencerError}; use sequencer::runtime::shutdown::RuntimeScope; use sequencer::storage::{SafeInputRange, Storage, StoredSafeInput}; use sequencer_core::api::WsTxMessage; -use sequencer_core::l2_tx::SequencedL2Tx; +use sequencer_core::history::{EraId, ExecutedInputCount, HistoryClaim, HistoryPolicyError}; use sequencer_core::user_op::{SignedUserOp, UserOp}; use sequencer_rust_client::SequencerClient; use tokio::sync::{mpsc, oneshot}; @@ -33,7 +34,7 @@ async fn ws_subscribe_streams_ordered_txs_from_offset_zero() { let Some(runtime) = start_test_server(db.path.as_str()).await else { return; }; - let url = ws_subscribe_url(runtime.addr, 0); + let url = ws_subscribe_url(&db.path, runtime.addr, 0); let (mut ws, _) = tokio::time::timeout(Duration::from_secs(5), connect_async(url)) .await .expect("timeout connecting websocket") @@ -45,27 +46,27 @@ async fn ws_subscribe_streams_ordered_txs_from_offset_zero() { shutdown_runtime(runtime).await; - // DB offsets (SQLite rowid) start at 1. - assert_ws_message_matches_tx(first, &expected[0], 1); - assert_ws_message_matches_tx(second, &expected[1], 2); + // Application offsets start at zero. + assert_ws_message_matches_tx(first, &expected[0], 0); + assert_ws_message_matches_tx(second, &expected[1], 1); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn ws_subscribe_resumes_from_given_offset() { let db = temp_db("ws-subscribe-resume"); seed_ordered_txs(db.path.as_str()); - // Resume from DB offset 1 — should get items with offset > 1. + // Resume inclusively at application input 1. let expected = load_ordered_l2_txs_page(db.path.as_str(), 1, 1); assert_eq!( expected.len(), 1, - "resume snapshot must contain one event at offset 2" + "resume snapshot must contain one event at offset 1" ); let Some(runtime) = start_test_server(db.path.as_str()).await else { return; }; - let url = ws_subscribe_url(runtime.addr, 1); + let url = ws_subscribe_url(&db.path, runtime.addr, 1); let (mut ws, _) = tokio::time::timeout(Duration::from_secs(5), connect_async(url)) .await .expect("timeout connecting websocket") @@ -76,7 +77,7 @@ async fn ws_subscribe_resumes_from_given_offset() { shutdown_runtime(runtime).await; - assert_ws_message_matches_tx(first, &expected[0], 2); + assert_ws_message_matches_tx(first, &expected[0], 1); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] @@ -90,7 +91,7 @@ async fn ws_subscribe_receives_live_events_after_subscribing() { }; // Subscribe at the current DB head to exercise live-only delivery. - let url = ws_subscribe_url(runtime.addr, base_offset); + let url = ws_subscribe_url(&db.path, runtime.addr, base_offset); let (mut ws, _) = tokio::time::timeout(Duration::from_secs(5), connect_async(url)) .await .expect("timeout connecting websocket") @@ -108,7 +109,7 @@ async fn ws_subscribe_receives_live_events_after_subscribing() { shutdown_runtime(runtime).await; - assert_ws_message_matches_tx(live, &expected[0], base_offset + 1); + assert_ws_message_matches_tx(live, &expected[0], base_offset); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] @@ -121,7 +122,7 @@ async fn ws_subscribe_fanout_delivers_live_event_to_multiple_subscribers() { return; }; - let url = ws_subscribe_url(runtime.addr, base_offset); + let url = ws_subscribe_url(&db.path, runtime.addr, base_offset); let (mut ws_a, _) = tokio::time::timeout(Duration::from_secs(5), connect_async(url.as_str())) .await .expect("timeout connecting websocket A") @@ -146,23 +147,21 @@ async fn ws_subscribe_fanout_delivers_live_event_to_multiple_subscribers() { shutdown_runtime(runtime).await; - assert_ws_message_matches_tx(event_a, &expected[0], base_offset + 1); - assert_ws_message_matches_tx(event_b, &expected[0], base_offset + 1); + assert_ws_message_matches_tx(event_a, &expected[0], base_offset); + assert_ws_message_matches_tx(event_b, &expected[0], base_offset); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn ws_subscribe_replies_with_pong_on_ping() { let db = temp_db("ws-subscribe-ping-pong"); seed_ordered_txs(db.path.as_str()); - // Use a far-future offset so this test validates ping/pong without - // interleaving replay/live tx frames. - let from_offset = u64::MAX; + let from_offset = ordered_l2_tx_count(&db.path); let Some(runtime) = start_test_server(db.path.as_str()).await else { return; }; - let url = ws_subscribe_url(runtime.addr, from_offset); + let url = ws_subscribe_url(&db.path, runtime.addr, from_offset); let (mut ws, _) = tokio::time::timeout(Duration::from_secs(5), connect_async(url)) .await .expect("timeout connecting websocket") @@ -189,11 +188,11 @@ async fn ws_subscribe_rejects_when_subscriber_limit_is_reached() { seed_ordered_txs(db.path.as_str()); let base_offset = ordered_l2_tx_count(db.path.as_str()); - let Some(runtime) = start_test_server_with_limits(db.path.as_str(), 1, 50_000).await else { + let Some(runtime) = start_test_server_with_limits(db.path.as_str(), 1).await else { return; }; - let url = ws_subscribe_url(runtime.addr, base_offset); + let url = ws_subscribe_url(&db.path, runtime.addr, base_offset); let (ws_a, _) = tokio::time::timeout(Duration::from_secs(5), connect_async(url.as_str())) .await .expect("timeout connecting websocket A") @@ -215,73 +214,30 @@ async fn ws_subscribe_rejects_when_subscriber_limit_is_reached() { } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn ws_subscribe_closes_when_catchup_window_exceeds_limit() { - let db = temp_db("ws-catchup-limit"); - seed_ordered_txs(db.path.as_str()); - append_drained_direct_input(db.path.as_str(), vec![0xbb]); - - let Some(runtime) = start_test_server_with_limits(db.path.as_str(), 64, 1).await else { - return; - }; - - let url = ws_subscribe_url(runtime.addr, 1); - let (mut ws, _) = tokio::time::timeout(Duration::from_secs(5), connect_async(url)) - .await - .expect("timeout connecting websocket") - .expect("connect websocket"); - - let frame = recv_raw_message(&mut ws).await; - match frame { - Message::Close(Some(close_frame)) => { - assert_eq!( - close_frame.code, - tokio_tungstenite::tungstenite::protocol::frame::coding::CloseCode::Policy - ); - let prefix = format!("{WS_CATCHUP_WINDOW_EXCEEDED_REASON}: live_start_offset="); - let reason = close_frame.reason.as_str(); - assert!( - reason.starts_with(WS_CATCHUP_WINDOW_EXCEEDED_REASON), - "close reason must retain the stable prefix: {reason}" - ); - let live_start_offset = reason - .strip_prefix(prefix.as_str()) - .expect("close reason carries live_start_offset") - .parse::() - .expect("live_start_offset is a u64"); - assert_eq!(live_start_offset, 3); - } - other => panic!("expected close frame for catch-up limit, got {other:?}"), - } - - drop(ws); - shutdown_runtime(runtime).await; -} - -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn ws_subscribe_allows_catchup_exactly_at_limit() { - let db = temp_db("ws-catchup-boundary"); - seed_ordered_txs(db.path.as_str()); - let expected = load_ordered_l2_txs_page(db.path.as_str(), 0, 2); - assert_eq!(expected.len(), 2, "seeded replay must contain two txs"); - - let Some(runtime) = start_test_server_with_limits(db.path.as_str(), 64, 2).await else { +async fn ws_subscribe_requires_a_valid_history_claim_before_upgrade() { + let db = temp_db("ws-history-claims"); + seed_ordered_txs(&db.path); + let Some(runtime) = start_test_server(&db.path).await else { return; }; - - let url = ws_subscribe_url(runtime.addr, 0); - let (mut ws, _) = tokio::time::timeout(Duration::from_secs(5), connect_async(url)) - .await - .expect("timeout connecting websocket") - .expect("connect websocket"); - - let first = recv_tx_message(&mut ws).await; - let second = recv_tx_message(&mut ws).await; - drop(ws); - + let client = SequencerClient::new(format!("http://{}", runtime.addr)).unwrap(); + let mut start = history_claim(&db.path, 3); + assert!(matches!(client.subscribe(start).await, + Err(sequencer_rust_client::SubscribeError::History(HistoryPolicyError::AheadOfHead { head })) if head.get() == 2)); + start.next_input = ExecutedInputCount::ZERO; + start.version.era_id = + EraId::from_bytes([0x11, 0, 0, 0, 0, 0, 0x40, 0, 0x80, 0, 0, 0, 0, 0, 0, 1]).unwrap(); + assert!(matches!( + client.subscribe(start).await, + Err(sequencer_rust_client::SubscribeError::History( + HistoryPolicyError::EraChanged { .. } + )) + )); + let legacy = connect_async(format!("ws://{}/ws/subscribe?from_offset=0", runtime.addr)).await; + assert!( + matches!(legacy, Err(tokio_tungstenite::tungstenite::Error::Http(response)) if response.status().as_u16() == 400) + ); shutdown_runtime(runtime).await; - - assert_ws_message_matches_tx(first, &expected[0], 1); - assert_ws_message_matches_tx(second, &expected[1], 2); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] @@ -293,7 +249,7 @@ async fn ws_subscribe_closes_on_oversized_inbound_message() { return; }; - let url = ws_subscribe_url(runtime.addr, u64::MAX); + let url = ws_subscribe_url(&db.path, runtime.addr, ordered_l2_tx_count(&db.path)); let (mut ws, _) = tokio::time::timeout(Duration::from_secs(5), connect_async(url)) .await .expect("timeout connecting websocket") @@ -320,6 +276,10 @@ async fn ws_subscribe_closes_on_oversized_inbound_message() { fn seed_ordered_txs(db_path: &str) { let mut storage = Storage::open(db_path).expect("open storage"); + crate::storage::test_helpers::pin_test_deployment_identity( + &mut storage, + Address::repeat_byte(0x7f), + ); let mut head = storage .initialize_open_state(0, SafeInputRange::empty_at(0)) .expect("initialize open state"); @@ -350,7 +310,7 @@ fn seed_ordered_txs(db_path: &str) { payload: vec![0xaa], block_number: 10, }], - Address::ZERO, + Address::repeat_byte(0x7f), &sequencer_core::protocol::ProtocolTiming { max_wait_blocks: sequencer_core::MAX_WAIT_BLOCKS, preemptive_margin_blocks: 75, @@ -386,7 +346,7 @@ fn append_drained_direct_input(db_path: &str, payload: Vec) { payload, block_number: safe_block, }], - Address::ZERO, + Address::repeat_byte(0x7f), &sequencer_core::protocol::ProtocolTiming { max_wait_blocks: sequencer_core::MAX_WAIT_BLOCKS, preemptive_margin_blocks: 75, @@ -426,13 +386,12 @@ fn snapshot_state_file(prefix: &std::path::Path) -> std::path::PathBuf { } async fn start_test_server(db_path: &str) -> Option { - start_test_server_with_limits(db_path, 64, 50_000).await + start_test_server_with_limits(db_path, 64).await } async fn start_test_server_with_limits( db_path: &str, ws_max_subscribers: usize, - ws_max_catchup_events: u64, ) -> Option { let listener = match tokio::net::TcpListener::bind("127.0.0.1:0").await { Ok(value) => value, @@ -454,8 +413,6 @@ async fn start_test_server_with_limits( L2TxFeedConfig { idle_poll_interval: Duration::from_millis(2), page_size: 64, - // Sentinel submitter: this fixture seeds no own-batch rows. - ..L2TxFeedConfig::new(alloy_primitives::Address::repeat_byte(0x7f)) }, ); let task = http::start_on_listener( @@ -465,7 +422,6 @@ async fn start_test_server_with_limits( tx_feed, ApiConfig { ws_max_subscribers, - ws_max_catchup_events, ..ApiConfig::new( Eip712Domain { name: None, @@ -540,35 +496,38 @@ fn decode_hex_prefixed(value: &str) -> Vec { alloy_primitives::hex::decode(value).expect("decode hex") } -fn ws_subscribe_url(addr: std::net::SocketAddr, from_offset: u64) -> String { +fn ws_subscribe_url(db_path: &str, addr: std::net::SocketAddr, from_offset: u64) -> String { let endpoint = format!("http://{addr}"); let client = SequencerClient::new(endpoint).expect("build sequencer client"); - client.ws_subscribe_url(from_offset) + client.ws_subscribe_url(history_claim(db_path, from_offset)) } fn ordered_l2_tx_count(db_path: &str) -> u64 { let mut storage = Storage::open_read_only(db_path).expect("open read-only storage"); storage - .ordered_l2_tx_head_offset() - .expect("query ordered l2 head offset") + .next_executed_input_count() + .expect("query application head") + .get() } -fn load_ordered_l2_txs_page(db_path: &str, from_offset: u64, limit: usize) -> Vec { +fn load_ordered_l2_txs_page( + db_path: &str, + from_offset: u64, + limit: usize, +) -> Vec { let mut storage = Storage::open_read_only(db_path).expect("open read-only storage"); storage - .ordered_l2_txs_page_from(from_offset, limit) + .canonical_history_page(history_claim(db_path, from_offset), limit) .expect("load ordered l2 tx page") - .into_iter() - .map(|row| row.tx) - .collect() + .rows } fn assert_ws_message_matches_tx( actual: WsTxMessage, - expected: &SequencedL2Tx, + expected: &ApplicationInputRow, expected_offset: u64, ) { - match (actual, expected) { + match (actual, &expected.context) { ( WsTxMessage::UserOp { offset, @@ -577,7 +536,7 @@ fn assert_ws_message_matches_tx( data, .. }, - SequencedL2Tx::UserOp(expected), + L2TxContext::UserOp { tx: expected, .. }, ) => { assert_eq!(offset, expected_offset); assert_eq!( @@ -595,7 +554,7 @@ fn assert_ws_message_matches_tx( payload, .. }, - SequencedL2Tx::Direct(expected), + L2TxContext::DirectInput { tx: expected, .. }, ) => { assert_eq!(offset, expected_offset); assert_eq!( @@ -613,3 +572,14 @@ fn assert_ws_message_matches_tx( } } } + +fn history_claim(db_path: &str, next: u64) -> HistoryClaim { + HistoryClaim { + version: Storage::open_read_only(db_path) + .unwrap() + .history_state() + .unwrap() + .version, + next_input: ExecutedInputCount::new(next), + } +} diff --git a/sequencer/src/lib.rs b/sequencer/src/lib.rs index 5c93c895..820ee3a3 100644 --- a/sequencer/src/lib.rs +++ b/sequencer/src/lib.rs @@ -38,4 +38,4 @@ pub use commands::config::{FlushConfig, RunConfig, SetupConfig}; pub use commands::error::CommandError; pub use commands::run::run; pub use harness::{Cli, Command, dispatch, run_command, run_main}; -pub use http::{ApiConfig, ApiError, WS_CATCHUP_WINDOW_EXCEEDED_REASON}; +pub use http::{ApiConfig, ApiError}; diff --git a/sequencer/src/recovery/mod.rs b/sequencer/src/recovery/mod.rs index 6ae3d556..f862ac79 100644 --- a/sequencer/src/recovery/mod.rs +++ b/sequencer/src/recovery/mod.rs @@ -90,8 +90,8 @@ pub enum RecoveryRefusalReason { /// assumes the opposite and is forbidden. #[error("canonical divergence at batch nonce {nonce}")] CanonicalDivergence { nonce: u64 }, - #[error("the completed setup has no finalized snapshot")] - MissingFinalizedSnapshot, + #[error("the completed setup has no recovery checkpoint")] + MissingRecoveryCheckpoint, #[error("post-sync recovery has no persisted safe head")] MissingSafeHead, /// The `EnsureOpenTip` transaction violated its open-Tip postcondition. @@ -165,9 +165,9 @@ fn refuse_local_terminal(facts: RecoveryInspection) -> Result<(), RecoveryError> RecoveryRefusalReason::CanonicalDivergence { nonce }, )); } - if !facts.has_finalized_snapshot { + if !facts.has_recovery_checkpoint { return Err(RecoveryError::refuse( - RecoveryRefusalReason::MissingFinalizedSnapshot, + RecoveryRefusalReason::MissingRecoveryCheckpoint, )); } Ok(()) @@ -470,8 +470,8 @@ fn classify_mutation(error: RecoveryMutationError) -> RecoveryError { RecoveryMutationError::CanonicalDivergence { nonce } => { RecoveryError::refuse(RecoveryRefusalReason::CanonicalDivergence { nonce }) } - RecoveryMutationError::MissingFinalizedSnapshot => { - RecoveryError::refuse(RecoveryRefusalReason::MissingFinalizedSnapshot) + RecoveryMutationError::MissingRecoveryCheckpoint => { + RecoveryError::refuse(RecoveryRefusalReason::MissingRecoveryCheckpoint) } RecoveryMutationError::MissingSafeHead => { RecoveryError::refuse(RecoveryRefusalReason::MissingSafeHead) @@ -627,14 +627,14 @@ mod tests { RecoveryMutationError::CanonicalDivergence { nonce: 7 }, )); assert_refuse(classify_mutation( - RecoveryMutationError::MissingFinalizedSnapshot, + RecoveryMutationError::MissingRecoveryCheckpoint, )); assert_refuse(classify_mutation(RecoveryMutationError::MissingSafeHead)); } fn admission_fixture( name: &str, - has_finalized_snapshot: bool, + has_recovery_checkpoint: bool, has_open_tip: bool, ) -> (crate::storage::test_helpers::TestDb, ProtocolTiming) { use crate::storage::test_helpers::{SENDER_A, default_protocol_timing, temp_db}; @@ -658,18 +658,23 @@ mod tests { .expect("seed fresh safe head"); let prefix = db._dir.path().join("finalized"); storage - .insert_initial_finalized_dump(&prefix, 0, 0, 0, 0) - .expect("seed finalized snapshot"); + .complete_baseline_setup( + &prefix, + sequencer_core::history::ExecutedInputCount::ZERO, + 0, + 0, + false, + ) + .expect("seed recovery checkpoint"); if has_open_tip { storage .initialize_open_state(0, storage::SafeInputRange::empty_at(0)) .expect("seed open Tip"); } - storage.complete_setup().expect("complete setup"); - if !has_finalized_snapshot { + if !has_recovery_checkpoint { storage .write(|tx| { - tx.execute("DELETE FROM finalized_snapshot", [])?; + tx.execute("DELETE FROM snapshots WHERE batch_index IS NULL", [])?; Ok(()) }) .expect("simulate post-setup snapshot loss"); @@ -835,8 +840,17 @@ mod tests { let block = protocol.danger_threshold(); let mut storage = storage::Storage::open_writer(&db.path).unwrap(); let mut head = storage.open_state().unwrap().unwrap(); + storage + .close_frame_only(&mut head, block, storage::SafeInputRange::empty_at(0)) + .unwrap(); storage.close_frame_and_batch(&mut head, block).unwrap(); + storage + .insert_batch_snapshot(&db._dir.path().join("batch0"), 0) + .unwrap(); storage.close_frame_and_batch(&mut head, block).unwrap(); + storage + .insert_batch_snapshot(&db._dir.path().join("batch1"), 1) + .unwrap(); storage .append_safe_inputs_with_timestamp( block, @@ -1027,14 +1041,14 @@ mod tests { let (db, protocol) = admission_fixture("admit-no-snapshot", false, true); let error = admit_runtime(&db.path, &protocol) - .expect_err("a missing finalized snapshot must refuse final admission"); + .expect_err("a missing recovery checkpoint must refuse final admission"); assert!(matches!( error, RecoveryError::Refuse(failure) if matches!( *failure, RecoveryFailure::PolicyRefusal( - RecoveryRefusalReason::MissingFinalizedSnapshot + RecoveryRefusalReason::MissingRecoveryCheckpoint ) ) )); diff --git a/sequencer/src/storage/convert.rs b/sequencer/src/storage/convert.rs index e8cb9536..be1e2376 100644 --- a/sequencer/src/storage/convert.rs +++ b/sequencer/src/storage/convert.rs @@ -106,7 +106,7 @@ pub(super) fn i64_to_u32(value: i64) -> u32 { // ── Query-bound conversion (saturating, by design) ──────────────────────── /// Saturating clamp for **untrusted or config-sourced** SQL query bounds: -/// WS `from_offset` cursors, page/count `LIMIT`s, and setup/recovery block +/// page/count `LIMIT`s and setup/recovery block /// predicates. The full `u64` range is legal input here, and clamping to /// `i64::MAX` preserves the comparison exactly — no SQLite `INTEGER` or rowid /// exceeds `i64::MAX`, so a past-the-end lower bound matches zero rows while a diff --git a/sequencer/src/storage/egress.rs b/sequencer/src/storage/egress.rs index 26615416..bb0f44ba 100644 --- a/sequencer/src/storage/egress.rs +++ b/sequencer/src/storage/egress.rs @@ -1,30 +1,18 @@ // (c) Cartesi and individual authors (see AUTHORS) // SPDX-License-Identifier: Apache-2.0 (see LICENSE) -//! Egress reader: ordered-L2-tx queries used by the WS feed and catch-up replay. -//! -//! Physical replay and canonical history use their respective `valid_*` views; -//! payload and provenance decoding is shared between both coordinates. +//! Application replay entries shared by catch-up and consumer history reads. use alloy_primitives::{Address, B256}; -use rusqlite::{Result, Row, params}; - -use super::Storage; -use super::convert::{i64_to_u32, i64_to_u64, saturating_query_bound}; -use super::queries::decode_l2_tx_row; +use rusqlite::{Result, Row}; use sequencer_core::history::ExecutedInputCount; -use sequencer_core::l2_tx::{DirectInput, SequencedL2Tx, ValidUserOp}; +use sequencer_core::l2_tx::{DirectInput, ValidUserOp}; + +use super::convert::{i64_to_u16, i64_to_u32, i64_to_u64}; -#[cfg_attr( - not(test), - expect( - dead_code, - reason = "Internal canonical reads await the coordinated WS cutover." - ) -)] mod canonical; +pub(crate) use canonical::HistoryReadError; -/// Application input and persisted context shared by both replay coordinates. #[derive(Debug, Clone)] pub(crate) enum L2TxContext { UserOp { @@ -36,212 +24,48 @@ pub(crate) enum L2TxContext { DirectInput { tx: DirectInput, input_index: u64, - safe_block: u64, batch_nonce: u64, block_timestamp: u64, transaction_hash: B256, }, } -/// Physical replay row, including rows that do not execute in the application. #[derive(Debug, Clone)] -pub(crate) struct OrderedL2TxRow { - pub(crate) offset: u64, - pub(crate) executed_input_offset: Option, +pub(crate) struct ApplicationInputRow { + pub(crate) offset: ExecutedInputCount, pub(crate) context: L2TxContext, } -impl OrderedL2TxRow { - fn into_replay_row(self) -> ReplayL2TxRow { - let (tx, frame_safe_block) = match self.context { - L2TxContext::UserOp { tx, safe_block, .. } => (SequencedL2Tx::UserOp(tx), safe_block), - L2TxContext::DirectInput { tx, safe_block, .. } => { - (SequencedL2Tx::Direct(tx), safe_block) - } - }; - ReplayL2TxRow { - db_offset: self.offset, - tx, - frame_safe_block, - executed_input_offset: self.executed_input_offset, - } - } -} - -/// One valid physical replay row with its canonical application attribution. -/// -/// The physical SQLite cursor and logical application offset are deliberately -/// named: they are different coordinates and callers must not infer their -/// meaning from tuple position. -#[derive(Debug, Clone)] -pub(crate) struct ReplayL2TxRow { - pub(crate) db_offset: u64, - pub(crate) tx: SequencedL2Tx, - pub(crate) frame_safe_block: u64, - pub(crate) executed_input_offset: Option, -} - -impl Storage { - /// Load a page of ordered L2 transactions starting after the given offset. - /// Each row names both its physical database cursor and optional logical - /// application offset. `frame_safe_block` is fed to user-op replay so the - /// app clock advances exactly as it did live (directs use their own block - /// number). `executed_input_offset` is `None` only for a physical row that - /// does not execute in the application. Callers advance with `db_offset` - /// rather than incrementing either coordinate. - pub(crate) fn ordered_l2_txs_page_from( - &mut self, - offset: u64, - limit: usize, - ) -> Result> { - self.ordered_l2_tx_rows_page_from(offset, limit) - .map(|rows| { - rows.into_iter() - .map(OrderedL2TxRow::into_replay_row) - .collect() - }) - } - - /// Load feed rows with all persisted execution and settlement context. - pub(crate) fn ordered_l2_tx_rows_page_from( - &mut self, - offset: u64, - limit: usize, - ) -> Result> { - if limit == 0 { - return Ok(Vec::new()); - } - - const SQL: &str = " - SELECT - s.offset, - CASE WHEN s.user_op_pos_in_frame IS NOT NULL THEN 0 ELSE 1 END AS kind, - CASE - WHEN s.user_op_pos_in_frame IS NOT NULL THEN u.sender - WHEN s.safe_input_index IS NOT NULL THEN d.sender - ELSE NULL - END AS sender, - CASE WHEN s.user_op_pos_in_frame IS NOT NULL THEN u.data ELSE NULL END AS data, - CASE WHEN s.user_op_pos_in_frame IS NOT NULL THEN f.fee ELSE NULL END AS fee, - CASE WHEN s.safe_input_index IS NOT NULL THEN d.payload ELSE NULL END AS payload, - CASE WHEN s.safe_input_index IS NOT NULL THEN d.block_number ELSE NULL END AS block_number, - f.safe_block, - b.nonce, - s.safe_input_index, - CASE WHEN s.user_op_pos_in_frame IS NOT NULL THEN u.nonce ELSE NULL END AS op_nonce, - CASE WHEN s.safe_input_index IS NOT NULL THEN d.block_timestamp ELSE NULL END AS block_timestamp, - CASE WHEN s.safe_input_index IS NOT NULL THEN d.transaction_hash ELSE NULL END AS transaction_hash, - e.executed_input_offset - FROM valid_sequenced_l2_txs s - LEFT JOIN user_ops u - ON u.batch_index = s.batch_index - AND u.frame_in_batch = s.frame_in_batch - AND u.pos_in_frame = s.user_op_pos_in_frame - LEFT JOIN frames f - ON f.batch_index = s.batch_index - AND f.frame_in_batch = s.frame_in_batch - LEFT JOIN safe_inputs d - ON d.safe_input_index = s.safe_input_index - LEFT JOIN batches b - ON b.batch_index = s.batch_index - LEFT JOIN executed_inputs e - ON e.sequenced_l2_tx_offset = s.offset - WHERE s.offset > ?1 - ORDER BY s.offset ASC - LIMIT ?2 - "; - let mut stmt = self.conn.prepare_cached(SQL)?; - let limit = u64::try_from(limit).unwrap_or(u64::MAX); - // Query bounds saturate by design: `offset` can be a client-supplied - // WS cursor and `limit` is config-sourced (see `saturating_query_bound`). - let rows = stmt.query_map( - params![ - saturating_query_bound(offset), - saturating_query_bound(limit) - ], - decode_ordered_l2_tx_row, - )?; - rows.collect::>>() - } - - /// Returns the maximum offset in `valid_sequenced_l2_txs`, or 0 if empty. - /// Used as the head cursor for feed subscribers. Shares the single - /// `valid_ordered_l2_tx_head` reader with the snapshot batch-close path, - /// so the feed cursor and the snapshot replay cursor can't drift. - pub fn ordered_l2_tx_head_offset(&mut self) -> Result { - super::queries::valid_ordered_l2_tx_head(&self.conn) - } - - /// Count broadcastable events with offset > `from_offset`, capped at `limit`. - /// - /// Used for catch-up window checks. Excludes batch-submitter direct - /// inputs — they are filtered before WS delivery, so the count reflects - /// what the client actually receives. - pub fn count_broadcastable_events_after( - &mut self, - from_offset: u64, - limit: u64, - batch_submitter_address: Address, - ) -> Result { - if limit == 0 { - return Ok(0); - } - - const SQL: &str = " - SELECT COUNT(*) FROM ( - SELECT 1 FROM valid_sequenced_l2_txs s - WHERE s.offset > ?1 - AND NOT (s.safe_input_index IS NOT NULL - AND EXISTS (SELECT 1 FROM safe_inputs si - WHERE si.safe_input_index = s.safe_input_index - AND si.sender = ?2)) - LIMIT ?3 - )"; - let value: i64 = self.conn.query_row( - SQL, - params![ - saturating_query_bound(from_offset), - batch_submitter_address.as_slice(), - saturating_query_bound(limit) - ], - |row| row.get(0), - )?; - Ok(i64_to_u64(value)) - } -} - -fn decode_ordered_l2_tx_row(row: &Row<'_>) -> Result { - let tx = decode_l2_tx_row( - row.get(1)?, - row.get(2)?, - row.get(3)?, - row.get(4)?, - row.get(5)?, - row.get(6)?, - ); +fn decode_application_input(row: &Row<'_>) -> Result { + let sender = Address::from_slice(row.get::<_, Vec>(2)?.as_slice()); let safe_block = i64_to_u64(row.get(7)?); let batch_nonce = i64_to_u64(row.get(8)?); - let context = match tx { - SequencedL2Tx::UserOp(tx) => L2TxContext::UserOp { - tx, + let context = match row.get::<_, i64>(1)? { + 0 => L2TxContext::UserOp { + tx: ValidUserOp { + sender, + data: row.get(3)?, + fee: i64_to_u16(row.get(4)?), + }, nonce: i64_to_u32(row.get(10)?), safe_block, batch_nonce, }, - SequencedL2Tx::Direct(tx) => L2TxContext::DirectInput { - tx, + 1 => L2TxContext::DirectInput { + tx: DirectInput { + sender, + payload: row.get(5)?, + block_number: i64_to_u64(row.get(6)?), + }, input_index: i64_to_u64(row.get(9)?), - safe_block, batch_nonce, block_timestamp: i64_to_u64(row.get(11)?), transaction_hash: B256::from_slice(row.get::<_, Vec>(12)?.as_slice()), }, + kind => panic!("invalid application input kind {kind}"), }; - Ok(OrderedL2TxRow { - offset: i64_to_u64(row.get(0)?), - executed_input_offset: row - .get::<_, Option>(13)? - .map(|value| ExecutedInputCount::new(i64_to_u64(value))), + Ok(ApplicationInputRow { + offset: ExecutedInputCount::new(i64_to_u64(row.get(0)?)), context, }) } diff --git a/sequencer/src/storage/egress/canonical.rs b/sequencer/src/storage/egress/canonical.rs index e84e07ff..8537293c 100644 --- a/sequencer/src/storage/egress/canonical.rs +++ b/sequencer/src/storage/egress/canonical.rs @@ -8,22 +8,25 @@ use sequencer_core::history::{ ExecutedInputCount, HistoryBounds, HistoryClaim, HistoryPolicyError, }; -use super::{L2TxContext, decode_ordered_l2_tx_row}; +use super::{ApplicationInputRow, decode_application_input}; use crate::storage::Storage; use crate::storage::convert::{saturating_query_bound, u64_to_i64}; use crate::storage::history::{next_executed_input_count_in, query_history_state}; -#[derive(Debug)] -pub(crate) struct CanonicalHistoryRow { - pub(crate) offset: ExecutedInputCount, - pub(crate) context: L2TxContext, -} - #[derive(Debug)] pub(crate) struct CanonicalHistoryPage { pub(crate) bounds: HistoryBounds, - pub(crate) rows: Vec, - pub(crate) next: HistoryClaim, + pub(crate) rows: Vec, + pub(crate) next_input: ExecutedInputCount, +} + +impl CanonicalHistoryPage { + pub(crate) fn next_claim(&self) -> HistoryClaim { + HistoryClaim { + version: self.bounds.version, + next_input: self.next_input, + } + } } #[derive(Debug, thiserror::Error)] @@ -59,11 +62,7 @@ impl Storage { fn history_bounds_in(conn: &Connection) -> rusqlite::Result { let state = query_history_state(conn)?; - let available_from = ExecutedInputCount::new( - state - .base_executed_input_count - .expect("application history base is unbound outside rebuild fill"), - ); + let available_from = ExecutedInputCount::new(state.base_executed_input_count); Ok(HistoryBounds { version: state.version, available_from, @@ -85,7 +84,7 @@ fn canonical_page_in( if expected_len > 0 { const SQL: &str = " SELECT - s.sequenced_l2_tx_offset, + s.offset, CASE WHEN s.user_op_pos_in_frame IS NOT NULL THEN 0 ELSE 1 END, CASE WHEN s.user_op_pos_in_frame IS NOT NULL THEN u.sender ELSE d.sender END, CASE WHEN s.user_op_pos_in_frame IS NOT NULL THEN u.data ELSE NULL END, @@ -97,9 +96,8 @@ fn canonical_page_in( s.safe_input_index, CASE WHEN s.user_op_pos_in_frame IS NOT NULL THEN u.nonce ELSE NULL END, CASE WHEN s.safe_input_index IS NOT NULL THEN d.block_timestamp ELSE NULL END, - CASE WHEN s.safe_input_index IS NOT NULL THEN d.transaction_hash ELSE NULL END, - s.executed_input_offset - FROM valid_executed_inputs s + CASE WHEN s.safe_input_index IS NOT NULL THEN d.transaction_hash ELSE NULL END + FROM application_inputs s LEFT JOIN user_ops u ON u.batch_index = s.batch_index AND u.frame_in_batch = s.frame_in_batch @@ -108,29 +106,23 @@ fn canonical_page_in( ON f.batch_index = s.batch_index AND f.frame_in_batch = s.frame_in_batch LEFT JOIN safe_inputs d ON d.safe_input_index = s.safe_input_index LEFT JOIN batches b ON b.batch_index = s.batch_index - WHERE s.executed_input_offset >= ?1 - ORDER BY s.executed_input_offset + WHERE s.offset >= ?1 + ORDER BY s.offset LIMIT ?2 "; let mut stmt = tx.prepare_cached(SQL)?; let mapped = stmt.query_map( params![u64_to_i64(from.get()), saturating_query_bound(expected_len)], - decode_ordered_l2_tx_row, + decode_application_input, )?; let mut expected = from; for row in mapped { let row = row?; - let offset = row - .executed_input_offset - .expect("canonical history row has no execution attribution"); assert_eq!( - offset, expected, + row.offset, expected, "canonical history page has an attribution gap" ); - rows.push(CanonicalHistoryRow { - offset, - context: row.context, - }); + rows.push(row); expected = expected .checked_next() .expect("canonical input count overflow"); @@ -143,12 +135,9 @@ fn canonical_page_in( } Ok(CanonicalHistoryPage { bounds, - next: HistoryClaim { - version: bounds.version, - next_input: from - .checked_add(expected_len) - .expect("page ends at or before head"), - }, + next_input: from + .checked_add(expected_len) + .expect("page ends at or before head"), rows, }) } diff --git a/sequencer/src/storage/egress/canonical/tests.rs b/sequencer/src/storage/egress/canonical/tests.rs index 580a28d8..63edf933 100644 --- a/sequencer/src/storage/egress/canonical/tests.rs +++ b/sequencer/src/storage/egress/canonical/tests.rs @@ -9,6 +9,8 @@ use tokio::sync::oneshot; use super::*; use crate::ingress::inclusion_lane::{IncludedUserOp, PendingUserOp}; +use crate::storage::L2TxContext; +use crate::storage::history::initialize_history_in; use crate::storage::test_helpers::{ SENDER_A, SENDER_B, default_protocol_timing, local_batch_payload, pin_test_deployment_identity, temp_db, @@ -111,7 +113,6 @@ fn canonical_pages_are_inclusive_and_preserve_context_without_batch_envelopes() safe_input_index: 0, executed_input_offset: ExecutedInputCount::new(1), }], - None, ) .unwrap(); storage @@ -121,10 +122,9 @@ fn canonical_pages_are_inclusive_and_preserve_context_without_batch_envelopes() let bounds = storage.history_bounds().unwrap(); assert_eq!(bounds.available_from, ExecutedInputCount::ZERO); assert_eq!(bounds.head, ExecutedInputCount::new(3)); - assert_eq!(storage.ordered_l2_txs_page_from(0, 10).unwrap().len(), 4); let first = storage.canonical_history_page(claim(bounds, 0), 2).unwrap(); assert_eq!(first.bounds, bounds); - assert_eq!(first.next, claim(bounds, 2)); + assert_eq!(first.next_claim(), claim(bounds, 2)); assert_eq!(first.rows.len(), 2); assert_eq!(first.rows[0].offset, ExecutedInputCount::ZERO); match &first.rows[0].context { @@ -146,7 +146,6 @@ fn canonical_pages_are_inclusive_and_preserve_context_without_batch_envelopes() L2TxContext::DirectInput { tx, input_index, - safe_block, batch_nonce, block_timestamp, transaction_hash: actual_hash, @@ -154,15 +153,14 @@ fn canonical_pages_are_inclusive_and_preserve_context_without_batch_envelopes() assert_eq!(tx.sender, SENDER_B); assert_eq!(tx.payload, vec![0x22]); assert_eq!(tx.block_number, 10); - assert_eq!( - (*input_index, *safe_block, *batch_nonce, *block_timestamp), - (0, 10, 1, 100) - ); + assert_eq!((*input_index, *batch_nonce, *block_timestamp), (0, 1, 100)); assert_eq!(*actual_hash, transaction_hash); } other => panic!("expected direct input, got {other:?}"), } - let last = storage.canonical_history_page(first.next, 2).unwrap(); + let last = storage + .canonical_history_page(first.next_claim(), 2) + .unwrap(); assert_eq!(last.rows.len(), 1); assert_eq!(last.rows[0].offset, ExecutedInputCount::new(2)); match &last.rows[0].context { @@ -178,17 +176,19 @@ fn canonical_pages_are_inclusive_and_preserve_context_without_batch_envelopes() } other => panic!("expected user op, got {other:?}"), } - assert_eq!(last.next, claim(bounds, 3)); - let tail = storage.canonical_history_page(last.next, 2).unwrap(); + assert_eq!(last.next_claim(), claim(bounds, 3)); + let tail = storage + .canonical_history_page(last.next_claim(), 2) + .unwrap(); assert!(tail.rows.is_empty()); - assert_eq!(tail.next, last.next); + assert_eq!(tail.next_claim(), last.next_claim()); let zero = storage.canonical_history_page(claim(bounds, 1), 0).unwrap(); assert!(zero.rows.is_empty()); - assert_eq!(zero.next, claim(bounds, 1)); + assert_eq!(zero.next_claim(), claim(bounds, 1)); } #[test] -fn recovery_refuses_old_claims_and_reuses_canonical_offsets_across_physical_holes() { +fn recovery_refuses_old_claims_and_reuses_canonical_offsets_after_suffix_replacement() { let db = temp_db("canonical-page-recovery"); let mut storage = Storage::open(&db.path).unwrap(); seed_aging_tip(&mut storage); @@ -223,27 +223,28 @@ fn recovery_refuses_old_claims_and_reuses_canonical_offsets_across_physical_hole .unwrap(); assert_eq!(page.rows.len(), 1); assert_eq!(page.rows[0].offset, ExecutedInputCount::new(1)); - assert_eq!(page.next.next_input, before.head); + assert_eq!(page.next_input, before.head); match &page.rows[0].context { L2TxContext::UserOp { tx, .. } => assert_eq!(tx.data, vec![0xcc]), other => panic!("expected replacement user op, got {other:?}"), } - let physical = storage.ordered_l2_txs_page_from(0, 10).unwrap(); - assert_eq!( - physical.iter().map(|row| row.db_offset).collect::>(), - vec![1, 3] - ); let audit_rows: i64 = storage .conn - .query_row("SELECT COUNT(*) FROM sequenced_l2_txs", [], |row| { - row.get(0) - }) + .query_row("SELECT COUNT(*) FROM user_ops", [], |row| row.get(0)) .unwrap(); assert_eq!(audit_rows, 3); + assert_eq!( + storage + .canonical_history_page(claim(recovered, 0), 10) + .unwrap() + .rows + .len(), + 2 + ); } #[test] -fn rebuilt_history_starts_at_its_absolute_base_and_excludes_padding() { +fn rebuilt_history_starts_at_its_absolute_base_without_padding() { let db = temp_db("canonical-page-rebuild"); let mut storage = Storage::initialize_for_command(&db.path, LifecycleCommand::Rebuild).unwrap(); storage @@ -265,10 +266,11 @@ fn rebuilt_history_starts_at_its_absolute_base_and_excludes_padding() { &default_protocol_timing(), ) .unwrap(); - storage.open_recovery_tip(10).unwrap(); - let physical_head = storage.valid_ordered_l2_tx_head().unwrap(); storage - .insert_initial_finalized_dump(&db._dir.path().join("recovered"), 10, physical_head, 41, 2) + .write(|tx| initialize_history_in(tx, ExecutedInputCount::new(41), 10)) + .unwrap(); + storage + .initialize_open_state(10, SafeInputRange::empty_at(2)) .unwrap(); let bounds = storage.history_bounds().unwrap(); assert_eq!(bounds.available_from, ExecutedInputCount::new(41)); @@ -300,14 +302,7 @@ fn rebuilt_history_starts_at_its_absolute_base_and_excludes_padding() { .unwrap(); assert_eq!(page.rows.len(), 1); assert_eq!(page.rows[0].offset, ExecutedInputCount::new(41)); - assert_eq!(page.next.next_input, ExecutedInputCount::new(42)); - let physical = storage.ordered_l2_txs_page_from(0, 10).unwrap(); - assert_eq!(physical.len(), 3); - assert!( - physical[..2] - .iter() - .all(|row| row.executed_input_offset.is_none()) - ); + assert_eq!(page.next_input, ExecutedInputCount::new(42)); } #[test] @@ -336,7 +331,7 @@ fn a_deep_backlog_can_be_read_in_small_bounded_pages() { .collect::>(), vec![from, from + 1] ); - assert_eq!(page.next, claim(bounds, from + 2)); + assert_eq!(page.next_claim(), claim(bounds, from + 2)); } } @@ -379,10 +374,12 @@ fn one_read_transaction_keeps_history_identity_and_rows_coherent_during_recovery fn tail_after_the_largest_sqlite_offset_is_empty_without_clamping() { let db = temp_db("canonical-page-sqlite-tail"); let mut storage = Storage::initialize_for_command(&db.path, LifecycleCommand::Rebuild).unwrap(); - storage.open_recovery_tip(0).unwrap(); let last = i64::MAX as u64; storage - .insert_initial_finalized_dump(&db._dir.path().join("recovered"), 0, 0, last, 0) + .write(|tx| initialize_history_in(tx, ExecutedInputCount::new(last), 0)) + .unwrap(); + storage + .initialize_open_state(0, SafeInputRange::empty_at(0)) .unwrap(); let mut head = storage.open_state().unwrap().unwrap(); storage @@ -396,15 +393,15 @@ fn tail_after_the_largest_sqlite_offset_is_empty_without_clamping() { assert_eq!(page.rows.len(), 1); assert_eq!(page.rows[0].offset, ExecutedInputCount::new(last)); let tail = storage - .canonical_history_page(page.next, usize::MAX) + .canonical_history_page(page.next_claim(), usize::MAX) .unwrap(); assert!(tail.rows.is_empty()); - assert_eq!(tail.next, page.next); + assert_eq!(tail.next_claim(), page.next_claim()); } #[test] #[should_panic(expected = "canonical history page has an attribution gap")] -fn an_interior_mapping_hole_fails_loud() { +fn an_interior_history_hole_fails_loud() { let db = temp_db("canonical-page-corrupt-attribution"); let mut storage = Storage::open(&db.path).unwrap(); let mut head = storage @@ -423,8 +420,8 @@ fn an_interior_mapping_hole_fails_loud() { storage .conn .execute_batch( - "DROP TRIGGER trg_protect_valid_executed_input_delete;\n\ - DELETE FROM executed_inputs WHERE executed_input_offset = 1;", + "DROP TRIGGER trg_protect_valid_application_input_delete;\n\ + DELETE FROM application_inputs WHERE offset = 1;", ) .unwrap(); let bounds = storage.history_bounds().unwrap(); diff --git a/sequencer/src/storage/history.rs b/sequencer/src/storage/history.rs index 7bb92047..65f24f37 100644 --- a/sequencer/src/storage/history.rs +++ b/sequencer/src/storage/history.rs @@ -1,42 +1,25 @@ // (c) Cartesi and individual authors (see AUTHORS) // SPDX-License-Identifier: Apache-2.0 (see LICENSE) -//! Current-era history metadata persisted as one SQLite singleton. +//! Immutable era baseline and current application-history generation. -use rusqlite::{Connection, Result, types::Type}; +#[cfg(test)] +use rusqlite::OptionalExtension; +use rusqlite::{Connection, Result, Transaction, params, types::Type}; use sequencer_core::history::{EraId, ExecutedInputCount, HistoryVersion, RecoveryGeneration}; use super::Storage; -use super::convert::{i64_to_u64, u64_to_i64}; +use super::convert::{i64_to_u64, now_unix_ms, u64_to_i64}; -/// Durable metadata for the history served by this database. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct HistoryState { pub version: HistoryVersion, pub era_created_at_ms: u64, - /// `None` only while an admitted cockroach rebuild has not yet registered - /// its recovered finalized application state. - pub base_executed_input_count: Option, - /// Exclusive `safe_inputs` cursor below which this era must never drain. - /// `None` has the same narrow pre-fill rebuild meaning as the application - /// base above; the two fields bind atomically. - pub base_safe_input_index: Option, -} - -/// One sparse attribution from SQLite's physical replay log to the canonical -/// application-history coordinate consumed by that row. -/// -/// Physical rows that do not execute in the application (our own submitted -/// batches and cockroach-root padding) deliberately have no mapping. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub(super) struct ExecutedInputMapping { - pub sequenced_l2_tx_offset: u64, - pub executed_input_offset: ExecutedInputCount, + pub base_executed_input_count: u64, + /// L1 prefix already accounted for by the era's initial application state. + pub base_safe_block: u64, } -/// One safe input from a drained physical range that actually executed in the -/// application. The range may also contain intentionally-unmapped rows, so the -/// caller supplies only these sparse attributions. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct DirectInputExecution { pub safe_input_index: u64, @@ -44,16 +27,10 @@ pub struct DirectInputExecution { } impl Storage { - /// Read the current era, recovery generation, and locally available base. pub fn history_state(&self) -> Result { query_history_state(&self.conn) } - /// Boundary before the next canonical application input executes. - /// - /// This is derived, never independently advanced: the maximum of the era's - /// recovered base and one past the greatest valid execution attribution. - /// Invalidating a suffix therefore rolls the value back automatically. pub fn next_executed_input_count(&mut self) -> Result { self.read(|tx| next_executed_input_count_in(tx)) } @@ -61,8 +38,8 @@ impl Storage { pub(super) fn query_history_state(conn: &Connection) -> Result { conn.query_row( - "SELECT era_id, era_created_at_ms, recovery_generation, \ - base_executed_input_count, base_safe_input_index \ + "SELECT era_id, era_created_at_ms, recovery_generation, + base_executed_input_count, base_safe_block FROM history_state WHERE singleton_id = 0", [], |row| { @@ -76,292 +53,144 @@ pub(super) fn query_history_state(conn: &Connection) -> Result { recovery_generation: RecoveryGeneration::new(i64_to_u64(row.get(2)?)), }, era_created_at_ms: i64_to_u64(row.get(1)?), - base_executed_input_count: row.get::<_, Option>(3)?.map(i64_to_u64), - base_safe_input_index: row.get::<_, Option>(4)?.map(i64_to_u64), + base_executed_input_count: i64_to_u64(row.get(3)?), + base_safe_block: i64_to_u64(row.get(4)?), }) }, ) } -/// Derive the next canonical application coordinate from durable facts. -pub(super) fn next_executed_input_count_in(conn: &Connection) -> Result { - let base = query_history_state(conn)? - .base_executed_input_count - .expect("application history base is unbound outside rebuild fill"); - let greatest_valid: Option = conn.query_row( - "SELECT MAX(executed_input_offset) FROM executed_inputs", - [], - |row| row.get(0), - )?; - let after_valid = greatest_valid.map_or(0, |offset| { - i64_to_u64(offset) - .checked_add(1) - .expect("executed input offset overflow: contract-impossible") - }); - Ok(ExecutedInputCount::new(base.max(after_valid))) -} - -/// Attach a sequence of explicit application offsets inside the physical-row -/// creation transaction. The schema enforces contiguous canonical offsets, -/// including offset reuse after suffix invalidation. -pub(super) fn attach_executed_inputs_in( - tx: &rusqlite::Transaction<'_>, - mappings: &[ExecutedInputMapping], +pub(super) fn initialize_history_in( + tx: &Transaction<'_>, + base: ExecutedInputCount, + base_safe_block: u64, ) -> Result<()> { - if mappings.is_empty() { + #[cfg(test)] + if let Some(existing) = query_history_state(tx).optional()? { + // Test fixtures initialize genesis when opening their schema. Production + // creates this row only with the complete durable baseline. + assert_eq!( + existing.base_executed_input_count, + base.get(), + "history base differs" + ); + assert_eq!( + existing.base_safe_block, base_safe_block, + "L1 prefix differs" + ); return Ok(()); } - - let mut stmt = tx.prepare_cached( - "INSERT INTO executed_inputs \ - (sequenced_l2_tx_offset, executed_input_offset) VALUES (?1, ?2)", - )?; - for mapping in mappings { - stmt.execute(rusqlite::params![ - u64_to_i64(mapping.sequenced_l2_tx_offset), - u64_to_i64(mapping.executed_input_offset.get()), - ])?; - } - Ok(()) -} - -/// Bind the era's locally-available application base and durable safe-input -/// drain floor to the snapshot that establishes them. Genesis is already -/// initialized to zero by the baseline migration; rebuild starts with both -/// `NULL` and reaches this function with the folded application's absolute -/// count plus the recovery root's exclusive safe-input cursor. -pub(super) fn bind_history_base_in( - tx: &rusqlite::Transaction<'_>, - base_executed_input_count: u64, - base_safe_input_index: u64, -) -> Result<()> { - let current = query_history_state(tx)?; - match ( - current.base_executed_input_count, - current.base_safe_input_index, - ) { - (Some(current_count), Some(current_safe_input_index)) => { - assert_eq!( - current_count, base_executed_input_count, - "history base differs from the initial finalized application state" - ); - assert_eq!( - current_safe_input_index, base_safe_input_index, - "safe-input floor differs from the initial finalized application state" - ); - return Ok(()); - } - (None, None) => {} - _ => unreachable!("history base pair cannot be partially bound"), - } - - let changed = tx.execute( - "UPDATE history_state \ - SET base_executed_input_count = ?1, base_safe_input_index = ?2 \ - WHERE singleton_id = 0 \ - AND base_executed_input_count IS NULL \ - AND base_safe_input_index IS NULL", - rusqlite::params![ - u64_to_i64(base_executed_input_count), - u64_to_i64(base_safe_input_index) + let mut bytes: [u8; EraId::BYTE_LEN] = + tx.query_row("SELECT randomblob(16)", [], |row| row.get(0))?; + bytes[6] = (bytes[6] & 0x0f) | 0x40; + bytes[8] = (bytes[8] & 0x3f) | 0x80; + let era = EraId::from_bytes(bytes).expect("UUID bits were set above"); + tx.execute( + "INSERT INTO history_state + (singleton_id, era_id, era_created_at_ms, recovery_generation, + base_executed_input_count, base_safe_block) VALUES (0, ?1, ?2, 0, ?3, ?4)", + params![ + era.as_bytes().as_slice(), + now_unix_ms(), + u64_to_i64(base.get()), + u64_to_i64(base_safe_block) ], )?; - if changed != 1 { - return Err(rusqlite::Error::StatementChangedRows(changed)); - } Ok(()) } -/// Durable lower bound for safe-input draining. A NULL floor is usable as zero -/// only while a rebuild's setup has not completed: that is the one interval -/// in which the recovery root must be populated before its cursor can be -/// bound. Everywhere else NULL is a storage invariant violation and fails -/// loud. -pub(super) fn safe_input_floor_in(conn: &Connection) -> Result { - let state = query_history_state(conn)?; - match state.base_safe_input_index { - Some(floor) => Ok(floor), - None => { - assert!( - state.base_executed_input_count.is_none(), - "history base pair cannot be partially bound" - ); - // A NULL floor exists only during a pre-completion rebuild - // fill: plain setup binds base 0 in its baseline transaction, and - // completion refuses while the base is NULL — so the completion - // fact alone decides legality (the black box is never read for - // decisions). - let setup_complete: bool = conn.query_row( - "SELECT EXISTS (SELECT 1 FROM setup_complete WHERE singleton_id = 0)", - [], - |row| row.get(0), - )?; - assert!( - !setup_complete, - "NULL safe-input floor outside pre-completion rebuild fill" - ); - Ok(0) - } - } +pub(super) fn next_executed_input_count_in(conn: &Connection) -> Result { + let base = query_history_state(conn)?.base_executed_input_count; + let greatest: Option = + conn.query_row("SELECT MAX(offset) FROM application_inputs", [], |row| { + row.get(0) + })?; + Ok(ExecutedInputCount::new(greatest.map_or(base, |offset| { + i64_to_u64(offset) + .checked_add(1) + .expect("application input count overflow") + }))) } -/// Advance the current era's soft-history reality by exactly one. The schema -/// independently rejects skips and rewrites; the caller composes this helper -/// into the same transaction as suffix invalidation and Tip reopening. -pub(super) fn advance_recovery_generation_in( - tx: &rusqlite::Transaction<'_>, -) -> Result { - let current: i64 = tx.query_row( - "SELECT recovery_generation FROM history_state WHERE singleton_id = 0", - [], - |row| row.get(0), - )?; +pub(super) fn advance_recovery_generation_in(tx: &Transaction<'_>) -> Result { + let current = query_history_state(tx)?.version.recovery_generation.get(); let next = current .checked_add(1) - .expect("recovery generation exhausted SQLite INTEGER: contract-impossible"); + .expect("recovery generation exhausted"); let changed = tx.execute( "UPDATE history_state SET recovery_generation = ?1 WHERE singleton_id = 0", - [next], + [u64_to_i64(next)], )?; if changed != 1 { return Err(rusqlite::Error::StatementChangedRows(changed)); } - Ok(RecoveryGeneration::new(i64_to_u64(next))) + Ok(RecoveryGeneration::new(next)) } #[cfg(test)] mod tests { use super::*; - use crate::storage::test_helpers::temp_db; - use crate::storage::{LifecycleCommand, Storage}; + use crate::storage::{LifecycleCommand, test_helpers::temp_db}; #[test] - fn history_schema_enforces_write_once_identity_and_base() { - let db = temp_db("history-write-once"); - let storage = Storage::open(db.path.as_str()).expect("initialize generic history"); - let original = storage.history_state().expect("read history"); + fn complete_baseline_is_immutable_and_survives_restart() { + let db = temp_db("history-baseline"); + let mut storage = + Storage::initialize_for_command(&db.path, LifecycleCommand::Rebuild).unwrap(); + assert!(matches!( + storage.history_state(), + Err(rusqlite::Error::QueryReturnedNoRows) + )); + storage + .write(|tx| initialize_history_in(tx, ExecutedInputCount::new(41), 70)) + .unwrap(); + let state = storage.history_state().unwrap(); + for sql in [ + "UPDATE history_state SET era_id = era_id", + "UPDATE history_state SET base_executed_input_count = 42", + "UPDATE history_state SET base_safe_block = 71", + "DELETE FROM history_state", + ] { + assert!(storage.conn.execute(sql, []).is_err(), "{sql}"); + } drop(storage); - - let conn = Storage::open_connection(db.path.as_str()).expect("open raw connection"); - assert!( - conn.execute( - "UPDATE history_state SET era_id = era_id WHERE singleton_id = 0", - [], - ) - .is_err(), - "era identity must reject even a same-value rewrite" - ); - assert!( - conn.execute( - "UPDATE history_state SET base_executed_input_count = 1 \ - WHERE singleton_id = 0", - [], - ) - .is_err(), - "the initialized genesis base must be immutable" - ); - assert!( - conn.execute( - "UPDATE history_state SET base_safe_input_index = 1 \ - WHERE singleton_id = 0", - [], - ) - .is_err(), - "the initialized genesis safe-input floor must be immutable" - ); - assert!( - conn.execute("DELETE FROM history_state WHERE singleton_id = 0", []) - .is_err(), - "the current era singleton must not be deletable" - ); - - let reopened = Storage::open_read_only(db.path.as_str()).expect("reopen"); - assert_eq!(reopened.history_state().expect("read history"), original); + let mut reopened = Storage::open(&db.path).unwrap(); + assert_eq!(reopened.history_state().unwrap(), state); + assert_eq!(reopened.next_executed_input_count().unwrap().get(), 41); } #[test] - fn rebuild_base_is_set_once_and_generation_advances_only_by_one() { - let db = temp_db("history-rebuild-transitions"); - let storage = Storage::initialize_for_command(db.path.as_str(), LifecycleCommand::Rebuild) - .expect("initialize rebuild"); + fn generation_advances_exactly_once_and_transaction_rollback_preserves_it() { + let db = temp_db("history-generation"); + let mut storage = Storage::open(&db.path).unwrap(); + let original = storage.history_state().unwrap(); + let result: Result<()> = storage.write(|tx| { + advance_recovery_generation_in(tx)?; + Err(rusqlite::Error::InvalidQuery) + }); + assert!(result.is_err()); + assert_eq!(storage.history_state().unwrap(), original); + storage.write(advance_recovery_generation_in).unwrap(); assert_eq!( storage .history_state() - .expect("read pending rebuild") - .base_executed_input_count, - None + .unwrap() + .version + .recovery_generation + .get(), + 1 ); - assert_eq!( - storage - .history_state() - .expect("read pending rebuild") - .base_safe_input_index, - None - ); - drop(storage); - - let conn = Storage::open_connection(db.path.as_str()).expect("open raw connection"); - assert!( - conn.execute( - "UPDATE history_state SET base_executed_input_count = 41 \ - WHERE singleton_id = 0", - [], - ) - .is_err(), - "the application base cannot bind without its safe-input floor" - ); - conn.execute( - "UPDATE history_state \ - SET base_executed_input_count = 41, base_safe_input_index = 7 \ - WHERE singleton_id = 0", - [], - ) - .expect("set rebuild base pair once"); assert!( - conn.execute( - "UPDATE history_state SET base_executed_input_count = 42 \ - WHERE singleton_id = 0", - [], - ) - .is_err(), - "rebuild base must not be rewritten" - ); - assert!( - conn.execute( - "UPDATE history_state SET base_safe_input_index = 8 \ - WHERE singleton_id = 0", - [], - ) - .is_err(), - "rebuild safe-input floor must not be rewritten" + storage + .conn + .execute("UPDATE history_state SET recovery_generation = 3", []) + .is_err() ); - - conn.execute( - "UPDATE history_state SET recovery_generation = 1 WHERE singleton_id = 0", - [], - ) - .expect("advance generation by one"); assert!( - conn.execute( - "UPDATE history_state SET recovery_generation = 3 WHERE singleton_id = 0", - [], - ) - .is_err(), - "generation must not skip" - ); - conn.execute( - "UPDATE history_state SET recovery_generation = 2 WHERE singleton_id = 0", - [], - ) - .expect("advance generation by one again"); - - let reopened = Storage::open_read_only(db.path.as_str()).expect("reopen"); - let state = reopened.history_state().expect("read history"); - assert_eq!(state.base_executed_input_count, Some(41)); - assert_eq!(state.base_safe_input_index, Some(7)); - assert_eq!( - state.version.recovery_generation, - RecoveryGeneration::new(2) + storage + .conn + .execute("UPDATE history_state SET recovery_generation = 0", []) + .is_err() ); } } diff --git a/sequencer/src/storage/ingress.rs b/sequencer/src/storage/ingress.rs index 3b73c9e5..095f1fa1 100644 --- a/sequencer/src/storage/ingress.rs +++ b/sequencer/src/storage/ingress.rs @@ -4,54 +4,48 @@ //! Inclusion-lane writer: opens the initial batch/frame, appends user-op chunks, //! and rotates frame/batch boundaries on the hot path. //! -//! The lane also reads `safe_inputs` (executed by the application) and the open +//! The lane also reads classified external directs and the open //! state (resumed on startup) — those reads live here too because they're driven //! by the lane's flow, not by an L1 ingress event. use std::path::Path; use alloy_primitives::Address; -use rusqlite::{Result, Transaction, params}; +use rusqlite::{OptionalExtension, Result, Transaction, params}; +#[cfg(test)] +use super::StoredSafeInput; +#[cfg(test)] +use super::convert::external_u64_to_i64; use super::convert::{ - external_u64_to_i64, from_unix_ms, i64_to_u64, now_unix_ms, saturating_query_bound, to_unix_ms, - u64_to_i64, + from_unix_ms, i64_to_u64, now_unix_ms, saturating_query_bound, to_unix_ms, u64_to_i64, }; -use super::history::{ExecutedInputMapping, attach_executed_inputs_in, safe_input_floor_in}; +use super::history::{next_executed_input_count_in, query_history_state}; use super::mutations::{ insert_new_batch, insert_open_frame, persist_frame_direct_sequence, - persist_frame_direct_sequence_derived, persist_frame_direct_sequence_physical_only, seal_batch, + persist_frame_direct_sequence_derived, seal_batch, }; use super::queries::{ current_safe_block_required, load_current_write_head, query_batch_policy, - query_latest_safe_input_index_exclusive, valid_ordered_l2_tx_head, + query_latest_safe_input_index_exclusive, }; use super::safe_accepted_batches::canonical_divergence_in; -use super::snapshot_dumps::{insert_pending_dump_in, promote_finalized_in}; +use super::snapshot_dumps::insert_batch_snapshot_in; use super::{ BatchPolicy, DirectInputExecution, ExecutedInputCount, SafeFrontierState, SafeInputFrontier, - SafeInputRange, Storage, StoredSafeInput, WriteHead, + SafeInputRange, Storage, WriteHead, }; use crate::ingress::inclusion_lane::{IncludedUserOp, PendingUserOp}; impl Storage { - /// Cursor for the next safe input to drain into a frame. Takes the maximum - /// of the era's durable drain floor and the highest already-drained - /// `safe_input_index` from valid (non-invalidated) `sequenced_l2_txs` rows - /// plus one. - /// - /// Using `MAX + 1` instead of `COUNT(*)` makes this robust against gaps: - /// when a batch is invalidated, those rows drop out of the view and the - /// cursor naturally rewinds, allowing the recovery batch to re-drain only - /// the invalidated suffix. The durable floor prevents a standard recovery - /// from crossing back into cockroach-root padding already represented by - /// the recovered application snapshot. + /// First L1 input beyond the latest surviving frame's accounted block, + /// bounded below by the immutable era prefix. pub fn next_undrained_safe_input_index(&mut self) -> Result { self.read(next_undrained_safe_input_index_in) } /// Resume the lane on startup. Returns `None` if storage is empty (caller - /// should follow up with [`Storage::initialize_open_state`]). + /// must establish the Tip through startup recovery). pub fn open_state(&mut self) -> Result> { self.read(load_current_write_head) } @@ -70,8 +64,7 @@ impl Storage { /// Bootstrap the very first batch + frame with explicit values, returning /// its loaded [`WriteHead`]. Asserts no open state exists. /// - /// Production opens the genesis Tip through the reducer's guarded - /// `EnsureOpenTip` phase, which derives `safe_block`/leading range from the + /// Production opens the genesis Tip through guarded startup recovery, which derives `safe_block`/leading range from the /// synced L1 view; this explicit form is kept for tests that seed a /// specific open state without a safe-head observation. #[cfg(test)] @@ -86,7 +79,7 @@ impl Storage { "open state already exists" ); let batch_index = insert_tip_rows(tx, Some(0), None, safe_block)?; - persist_frame_direct_sequence_physical_only(tx, batch_index, 0, leading_direct_range)?; + persist_frame_direct_sequence_derived(tx, batch_index, 0, leading_direct_range)?; Ok(load_current_write_head(tx)?.expect("genesis tip just inserted")) }) } @@ -122,23 +115,9 @@ impl Storage { }) } - /// Open the cockroach-recovery root tip: a fresh anchored tip (`batch_index` - /// 0, parentless, nonce = the batch-tree anchor `N'`) whose first frame sits - /// at the checkpoint stop block `C` and leads exactly the directs the fold - /// folded into `S'` — those with `block_number <= C`. - /// - /// Distinct from [`open_fresh_tip_in_tx`] (reached in production through - /// the reducer's guarded `EnsureOpenTip` phase), which drains the **whole** - /// synced table at the live safe head. That is - /// correct for genesis, but wrong here: `setup --recovery` resyncs to the - /// live safe head `H1`, which is normally **past** `C`, and the fold only - /// folded `<= C` into `S'`. Draining `(C, H1]` into this tip would sequence - /// those directs as "already executed" (advancing the cursor / snapshot - /// `l2_tx_index` past them) even though `S'` never executed them — they would - /// then be skipped by catch-up and never re-led, vanishing from local state - /// while the scheduler drains them on-chain (divergence). Capping the drain - /// at `C` leaves `(C, H1]` undrained, so `run`'s lane leads and executes them - /// exactly once as the safe frontier advances `C -> H1`. + /// Open the rebuilt root at the baseline's L1 stop block. Its prefix is + /// represented by the baseline artifact and contributes no replay rows. + #[cfg(test)] pub(crate) fn open_recovery_tip(&mut self, stop_block: u64) -> Result<()> { external_u64_to_i64(stop_block, "recovery checkpoint block")?; self.write(|tx| open_recovery_tip_in_tx(tx, stop_block)) @@ -165,10 +144,51 @@ impl Storage { }) } + pub(crate) fn fill_direct_inputs( + &mut self, + range: SafeInputRange, + out: &mut Vec, + ) -> Result<()> { + out.clear(); + if range.is_empty() { + return Ok(()); + } + let identity = super::l1_inputs::query_deployment_identity(&self.conn)? + .ok_or(rusqlite::Error::QueryReturnedNoRows)?; + let mut statement = self.conn.prepare_cached( + "SELECT safe_input_index, sender, payload, block_number FROM safe_inputs + WHERE safe_input_index >= ?1 AND safe_input_index < ?2 AND sender != ?3 + ORDER BY safe_input_index", + )?; + let rows = statement.query_map( + params![ + u64_to_i64(range.start()), + u64_to_i64(range.end()), + identity.batch_submitter_address.as_slice() + ], + |row| { + let sender: Vec = row.get(1)?; + Ok(super::StoredDirectInput { + safe_input_index: i64_to_u64(row.get(0)?), + input: sequencer_core::l2_tx::DirectInput { + sender: Address::from_slice(&sender), + payload: row.get(2)?, + block_number: i64_to_u64(row.get(3)?), + }, + }) + }, + )?; + for row in rows { + out.push(row?); + } + Ok(()) + } + /// Replace `out`'s contents with the safe-input rows in `range`. Asserts /// contiguity — gaps in `safe_input_index` are a bug, not a runtime /// condition. - pub fn fill_safe_inputs( + #[cfg(test)] + pub(crate) fn fill_safe_inputs( &mut self, range: SafeInputRange, out: &mut Vec, @@ -298,31 +318,22 @@ impl Storage { Ok(()) } - /// Rotate to the next frame, atomically attaching the drain and its - /// execution offsets and optionally promoting `(batch_nonce, inclusion_block)`. - /// Promotion must share the drain transaction: otherwise a restart could - /// replay the observation and try to promote its already-deleted pending row. + /// Commit frame advancement together with its application inputs. pub fn close_frame_only_with_executions( &mut self, head: &mut WriteHead, next_safe_block: u64, leading_direct_range: SafeInputRange, executions: &[DirectInputExecution], - promotion: Option<(u64, u64)>, ) -> Result<()> { let policy = self.write(|tx| { - let policy = - close_frame_in(tx, head, next_safe_block, leading_direct_range, executions)?; - if let Some((max_nonce, inclusion_block)) = promotion { - promote_finalized_in(tx, max_nonce, inclusion_block)?; - } - Ok(policy) + close_frame_in(tx, head, next_safe_block, leading_direct_range, executions) })?; head.advance_frame(policy, next_safe_block); Ok(()) } - /// Physical-only fixture frame rotation. Production must supply explicit + /// Fixture frame rotation with derived application offsets. Production must supply explicit /// execution attributions through /// [`Storage::close_frame_only_with_executions`]. #[cfg(test)] @@ -332,36 +343,15 @@ impl Storage { next_safe_block: u64, leading_direct_range: SafeInputRange, ) -> Result<()> { - let policy = self.write(|tx| { - close_frame_physical_only_in(tx, head, next_safe_block, leading_direct_range) - })?; - head.advance_frame(policy, next_safe_block); - Ok(()) - } - - /// Physical-only fixture form of the atomic drain + promotion operation. - #[cfg(test)] - pub fn close_frame_only_promoting( - &mut self, - head: &mut WriteHead, - next_safe_block: u64, - leading_direct_range: SafeInputRange, - max_nonce: u64, - inclusion_block: u64, - ) -> Result<()> { - let policy = self.write(|tx| { - let policy = - close_frame_physical_only_in(tx, head, next_safe_block, leading_direct_range)?; - promote_finalized_in(tx, max_nonce, inclusion_block)?; - Ok(policy) - })?; + let policy = self + .write(|tx| close_frame_derived_in(tx, head, next_safe_block, leading_direct_range))?; head.advance_frame(policy, next_safe_block); Ok(()) } /// Close the current batch and open a fresh one with its first frame, - /// without registering a pending snapshot. Test-only: production closes - /// through [`Storage::close_frame_and_batch_with_pending_dump`], which + /// without registering a snapshot. Test-only: production closes + /// through [`Storage::close_frame_and_batch_with_snapshot`], which /// registers the snapshot row in the same transaction (I7). /// /// Atomically: seal the current Tip (sets `sealed_at_ms`), insert the new @@ -385,48 +375,29 @@ impl Storage { Ok(()) } - /// Close the current batch, open a fresh one with its first frame, and - /// in the same transaction register the pending snapshot for the batch - /// being sealed. The production batch close. - /// - /// The caller must have already created the dump on disk at - /// `dump_prefix` (filesystem-first, outside this tx). That ordering - /// gives clean failure semantics: - /// - `create_dump` fails → caller never reaches here → batch stays - /// the open Tip → retried next pass. - /// - this tx fails → seal rolls back → no sealed-without-dump batch, - /// only an orphan directory the startup sweep reaps. - /// - this tx commits → the sealed batch always has a promotable - /// `pending_snapshots` row, closing the "seal succeeds, snapshot - /// fails, promotion wedges on `QueryReturnedNoRows` forever" gap. - /// - /// `nonce` is the closing batch's nonce (assigned at open, so known - /// before the seal). `l2_tx_index` is the global valid replay head - /// the caller read when writing the dump's `info.toml` — correct - /// even when the closing batch is empty. - pub fn close_frame_and_batch_with_pending_dump( + /// The artifact is durable before sealing and registering its snapshot in + /// one transaction. A failed commit leaves only an unreferenced artifact. + pub fn close_frame_and_batch_with_snapshot( &mut self, head: &mut WriteHead, next_safe_block: u64, dump_dir: &Path, - nonce: u64, - l2_tx_index: u64, + batch_index: u64, executed_input_count: ExecutedInputCount, ) -> Result<()> { + assert_eq!( + batch_index, head.batch_index, + "snapshot belongs to another batch" + ); let (next_batch_index, now_ms, policy) = self.write(|tx| { - // The lane is the single writer and nothing sequences between - // the caller's head read and this close, so the head cannot - // have moved. Assert it: the snapshot row and the dump's - // info.toml must record the same cursor. - let head_now = valid_ordered_l2_tx_head(tx)?; assert_eq!( - head_now, l2_tx_index, - "replay head moved between dump creation and batch close" + next_executed_input_count_in(tx)?, + executed_input_count, + "application count changed between dump creation and batch close" ); - let (next_batch_index, now_ms, policy) = - seal_and_open_next_batch(tx, head.batch_index, next_safe_block)?; - insert_pending_dump_in(tx, dump_dir, nonce, l2_tx_index, executed_input_count)?; - Ok((next_batch_index, now_ms, policy)) + let result = seal_and_open_next_batch(tx, head.batch_index, next_safe_block)?; + insert_batch_snapshot_in(tx, dump_dir, batch_index, executed_input_count)?; + Ok(result) })?; head.move_to_next_batch( next_batch_index, @@ -442,17 +413,9 @@ impl Storage { } } -/// Insert a fresh open batch (the Tip) and its first empty frame inside `tx`. -/// The caller immediately persists the leading direct range through either the -/// production attributed path or the explicit cockroach/test padding path. -/// Lineage is the caller's: `batch_index_opt = Some(0)` forces the genesis index, `None` -/// auto-assigns the PK; `parent = None` roots a nonce-0 batch (genesis or a -/// fully-torn refork), else it inherits `parent.nonce + 1`. The single-Tip -/// invariant is enforced by the `ux_single_valid_tip` partial index. -/// -/// Does **not** build a [`WriteHead`] — callers load it via -/// `load_current_write_head`, so the head has one constructor. Returns the new -/// `batch_index`. +/// Insert the Tip and its first frame. Callers separately attach the complete +/// leading application range, except for a recovery baseline's empty root. +/// Returns its local identity; callers load `WriteHead` through the shared reader. fn insert_tip_rows( tx: &Transaction<'_>, batch_index_opt: Option, @@ -477,11 +440,11 @@ fn insert_tip_rows( /// draining all currently-undrained safe inputs into its first frame. /// /// One mechanism, two callers with distinct intents (each keeps its own guard): -/// the reducer's guarded `EnsureOpenTip` phase (genesis / first startup) and +/// startup recovery's guarded Tip creation (genesis / first startup) and /// recovery's cascade (reopening the Tip it just invalidated, atomically — see /// `storage/recovery.rs`). Lineage is derived from the tree: `parent` is the /// highest-indexed valid batch — `None` when the valid path is empty (genesis, -/// or a fully-torn cascade), rooting a nonce-0 batch; `batch_index` is the +/// or a fully-torn cascade), rooting a batch at the deployment anchor; `batch_index` is the /// explicit genesis `0` only when the `batches` table is empty, otherwise the /// monotonic PK (so recovery batches keep climbing and indices are never /// reused). @@ -507,13 +470,8 @@ pub(super) fn open_fresh_tip_in_tx(tx: &Transaction<'_>) -> Result<()> { ) } -/// The shared tail of [`open_fresh_tip_in_tx`] and [`open_recovery_tip_in_tx`]: -/// open the single valid Tip draining `[next_undrained, drain_upper)`, framed at -/// `safe_block`, with the given lineage. The two callers differ only in -/// `drain_upper` — the live safe-input head on the fresh path vs the `<= C` cap -/// on the recovery path (the load-bearing `(C, H1]` difference) — plus -/// `safe_block` and lineage, which they pass explicitly so the cap rule stays -/// visible at each call site rather than hidden in one branch. +/// Capture the unaccounted range before creating its new frame, then attribute +/// its external directs. Catch-up executes these rows before runtime admission. fn insert_draining_tip_with_executions( tx: &Transaction<'_>, batch_index: Option, @@ -528,69 +486,51 @@ fn insert_draining_tip_with_executions( Ok(()) } -/// See [`Storage::open_recovery_tip`]. Like [`open_fresh_tip_in_tx`] but the -/// frame's safe block is the checkpoint stop block `C` and the leading drain -/// range is capped at the `<= C` directs (not the whole synced table), so the -/// `(C, H1]` directs the resync pulled past `C` stay undrained for `run`. -fn open_recovery_tip_in_tx(tx: &Transaction<'_>, stop_block: u64) -> Result<()> { - debug_assert!( +/// The folded prefix is represented by the baseline, so the root contains no +/// replay entries for it. Inputs beyond the stop block remain unaccounted. +pub(super) fn open_recovery_tip_in_tx(tx: &Transaction<'_>, stop_block: u64) -> Result<()> { + assert!( load_current_write_head(tx)?.is_none(), - "recovery tip opened over existing open state" - ); - // Fresh DB after wipe: batch_index 0, parentless (rooted at the anchor via - // `compute_next_nonce(None)` -> `N'`); drain capped at the `<= C` safe - // inputs. - // - // That range is sender-unfiltered: it includes the `<= C` batch-submitter - // rows alongside the user directs, exactly as the fresh / genesis tip drains - // its whole `[next_undrained, latest)` span. Correct because the leading - // range is *sequenced, not executed* — it only advances the replay cursor so - // `run`'s catch-up (`offset > l2_tx_index`) skips the rows already in `S'`. - // The `sender != batch_submitter` drop belongs to the *fold's* seed filter - // (those batches were folded into `S'` as batches, not directs); it is not a - // cursor concern, so it deliberately does not reappear here. - let leading_direct_range = SafeInputRange::new( - next_undrained_safe_input_index_in(tx)?, - safe_input_index_exclusive_through_block_in(tx, stop_block)?, + "recovery tip already exists" ); - // These rows are cursor padding for state already represented by the - // recovered snapshot. They permanently remain outside `executed_inputs`. - let batch_index = insert_tip_rows(tx, Some(0), None, stop_block)?; - persist_frame_direct_sequence_physical_only(tx, batch_index, 0, leading_direct_range)?; + insert_tip_rows(tx, Some(0), None, stop_block)?; Ok(()) } -/// Exclusive `safe_input_index` boundary separating directs at `block_number <= -/// block` from those past it. `safe_input_index` is dense and assigned in -/// block-ascending ingest order, so the `<= block` rows occupy `[0, count)` and -/// this count is the boundary. fn safe_input_index_exclusive_through_block_in(tx: &Transaction<'_>, block: u64) -> Result { - let boundary: i64 = tx.query_row( - "SELECT COUNT(*) FROM safe_inputs WHERE block_number <= ?1", - params![saturating_query_bound(block)], - |row| row.get(0), - )?; - Ok(i64_to_u64(boundary)) + let last: Option = tx + .query_row( + "SELECT safe_input_index FROM safe_inputs WHERE block_number <= ?1 + ORDER BY block_number DESC, safe_input_index DESC LIMIT 1", + [saturating_query_bound(block)], + |row| row.get(0), + ) + .optional()?; + Ok(last.map_or(0, |index| { + i64_to_u64(index) + .checked_add(1) + .expect("safe input index overflow") + })) } -/// Maximum of the durable era floor and `MAX(safe_input_index) + 1` over valid -/// drained rows (or 0 if none), inside `tx`. The valid attribution may rewind -/// when a batch is invalidated, but never below inputs already represented by -/// the cockroach-recovered base snapshot. fn next_undrained_safe_input_index_in(tx: &Transaction<'_>) -> Result { - const SQL: &str = " - SELECT COALESCE(MAX(safe_input_index) + 1, 0) - FROM valid_sequenced_l2_txs - WHERE safe_input_index IS NOT NULL - "; - let value: i64 = tx.query_row(SQL, [], |row| row.get(0))?; - Ok(safe_input_floor_in(tx)?.max(i64_to_u64(value))) + let floor = query_history_state(tx)?.base_safe_block; + let latest_frame: Option = tx + .query_row( + "SELECT safe_block FROM frames + WHERE batch_index = (SELECT MAX(batch_index) FROM valid_batches) + ORDER BY frame_in_batch DESC LIMIT 1", + [], + |row| row.get(0), + ) + .optional()?; + safe_input_index_exclusive_through_block_in(tx, floor.max(latest_frame.map_or(0, i64_to_u64))) } /// Seal the current Tip and open the successor batch's first frame, in `tx`. /// /// Shared by [`Storage::close_frame_and_batch`] and -/// [`Storage::close_frame_and_batch_with_pending_dump`] so the seal ordering +/// [`Storage::close_frame_and_batch_with_snapshot`] so the seal ordering /// invariant lives in one place: seal first (which frees the old row from the /// `ux_single_valid_tip` partial index), then insert the successor as the new /// Tip. Returns the new batch index, the close timestamp, and the sampled @@ -600,6 +540,11 @@ fn seal_and_open_next_batch( closing_batch_index: u64, next_safe_block: u64, ) -> Result<(u64, i64, BatchPolicy)> { + let current = load_current_write_head(tx)?.expect("a batch close requires the Tip"); + assert_eq!( + current.safe_block, next_safe_block, + "batch closure cannot advance the frame clock" + ); let now_ms = now_unix_ms(); // Batch policy is sampled here: the derived fee is committed to the newly // opened frame, and the batch size target is stored on the write head. @@ -627,8 +572,8 @@ fn seal_and_open_next_batch( /// Rotate to the next frame inside the current batch, in `tx`: open the /// successor frame (fresh fee/safe-block) and sequence the drained safe-input -/// range into it. Shared by the attributed production close/promote paths and -/// their physical-only test siblings so the frame-rotation invariant lives in +/// range into it. Shared by the attributed production frame-close path and +/// their application-history test siblings so the frame-rotation invariant lives in /// one place. Returns the sampled policy for the caller to apply to the /// in-memory write head. fn close_frame_in( @@ -638,6 +583,14 @@ fn close_frame_in( leading_direct_range: SafeInputRange, executions: &[DirectInputExecution], ) -> Result { + assert_eq!( + leading_direct_range, + SafeInputRange::new( + next_undrained_safe_input_index_in(tx)?, + safe_input_index_exclusive_through_block_in(tx, next_safe_block)?, + ), + "frame advancement must account for its complete L1 interval" + ); let (policy, next_frame_in_batch) = open_successor_frame_in(tx, head, next_safe_block)?; persist_frame_direct_sequence( tx, @@ -650,14 +603,14 @@ fn close_frame_in( } #[cfg(test)] -fn close_frame_physical_only_in( +fn close_frame_derived_in( tx: &Transaction<'_>, head: &WriteHead, next_safe_block: u64, leading_direct_range: SafeInputRange, ) -> Result { let (policy, next_frame_in_batch) = open_successor_frame_in(tx, head, next_safe_block)?; - persist_frame_direct_sequence_physical_only( + persist_frame_direct_sequence_derived( tx, head.batch_index, next_frame_in_batch, @@ -671,6 +624,10 @@ fn open_successor_frame_in( head: &WriteHead, next_safe_block: u64, ) -> Result<(BatchPolicy, u32)> { + assert!( + next_safe_block >= head.safe_block, + "frame clock cannot regress" + ); let now_ms = now_unix_ms(); let policy = query_batch_policy(tx)?; let next_frame_in_batch = head @@ -688,8 +645,7 @@ fn open_successor_frame_in( Ok((policy, next_frame_in_batch)) } -/// Insert user ops into `user_ops`. The `trg_sequence_user_op` trigger then -/// appends the matching `sequenced_l2_txs` row for each insert. +/// Fixture insertion of source operations and their application positions. #[cfg(test)] fn insert_user_ops_batch( tx: &Transaction<'_>, @@ -698,13 +654,21 @@ fn insert_user_ops_batch( frame_pos_start: u32, user_ops: &[PendingUserOp], ) -> Result<()> { + let mut next = next_executed_input_count_in(tx)?; insert_user_op_iter( tx, batch_index, frame_in_batch, frame_pos_start, user_ops.iter(), - ) + )?; + for position in 0..user_ops.len() { + tx.execute("INSERT INTO application_inputs (offset,batch_index,frame_in_batch,user_op_pos_in_frame) VALUES (?1,?2,?3,?4)", + params![u64_to_i64(next.get()),u64_to_i64(batch_index),i64::from(frame_in_batch), + i64::from(frame_pos_start.checked_add(u32::try_from(position).unwrap()).unwrap())])?; + next = next.checked_next().expect("input count overflow"); + } + Ok(()) } fn insert_user_op_iter<'a>( @@ -742,10 +706,7 @@ fn insert_user_op_iter<'a>( Ok(()) } -/// Persist one included user-op chunk and attach every explicit application -/// offset before the transaction can commit. The trigger-created physical rows -/// are selected in frame-position order once per chunk, keeping the hot path to -/// one attribution read rather than one lookup per operation. +/// User-op source and application position become durable before acknowledgement. fn insert_executed_user_ops_batch( tx: &Transaction<'_>, batch_index: u64, @@ -760,64 +721,22 @@ fn insert_executed_user_ops_batch( frame_pos_start, user_ops.iter().map(|item| &item.pending), )?; - if user_ops.is_empty() { - return Ok(()); - } - - let chunk_len = u32::try_from(user_ops.len()) - .expect("user-op chunk length exceeds u32: contract-impossible"); - let frame_pos_end = frame_pos_start - .checked_add(chunk_len) - .expect("user-op position overflow: contract-impossible"); let mut stmt = tx.prepare_cached( - "SELECT offset, user_op_pos_in_frame \ - FROM sequenced_l2_txs \ - WHERE batch_index = ?1 \ - AND frame_in_batch = ?2 \ - AND user_op_pos_in_frame >= ?3 \ - AND user_op_pos_in_frame < ?4 \ - ORDER BY user_op_pos_in_frame ASC", + "INSERT INTO application_inputs (offset, batch_index, frame_in_batch, user_op_pos_in_frame) + VALUES (?1, ?2, ?3, ?4)", )?; - let rows = stmt.query_map( - params![ + for (position, item) in user_ops.iter().enumerate() { + let position = frame_pos_start + .checked_add(u32::try_from(position).expect("chunk fits u32")) + .expect("user-op position overflow"); + stmt.execute(params![ + u64_to_i64(item.executed_input_offset.get()), u64_to_i64(batch_index), i64::from(frame_in_batch), - i64::from(frame_pos_start), - i64::from(frame_pos_end), - ], - |row| Ok((row.get::<_, i64>(0)?, row.get::<_, i64>(1)?)), - )?; - let persisted = rows.collect::>>()?; - drop(stmt); - assert_eq!( - persisted.len(), - user_ops.len(), - "user-op physical row count differs from included execution count" - ); - - let mut mappings = Vec::with_capacity(persisted.len()); - for (offset, ((physical_offset, position), executed_input_offset)) in persisted - .into_iter() - .zip(user_ops.iter().map(|item| item.executed_input_offset)) - .enumerate() - { - let offset = - u32::try_from(offset).expect("user-op chunk offset exceeds u32: contract-impossible"); - assert_eq!( - i64_to_u64(position), - u64::from( - frame_pos_start - .checked_add(offset) - .expect("user-op position overflow: contract-impossible") - ), - "user-op physical rows are not contiguous in frame order" - ); - mappings.push(ExecutedInputMapping { - sequenced_l2_tx_offset: i64_to_u64(physical_offset), - executed_input_offset, - }); + i64::from(position) + ])?; } - attach_executed_inputs_in(tx, &mappings) + Ok(()) } #[cfg(test)] @@ -860,20 +779,6 @@ mod tests { } } - fn physical_user_op_offsets(storage: &Storage) -> Vec { - storage - .conn - .prepare( - "SELECT offset FROM sequenced_l2_txs \ - WHERE user_op_pos_in_frame IS NOT NULL ORDER BY offset", - ) - .expect("prepare physical user-op query") - .query_map([], |row| row.get(0)) - .expect("query physical user-op offsets") - .collect::>>() - .expect("collect physical user-op offsets") - } - fn pin_deployment_identity(storage: &mut Storage, batch_submitter_address: Address) { storage .load_or_insert_deployment_identity(DeploymentIdentity { @@ -887,6 +792,73 @@ mod tests { .expect("pin deployment identity"); } + #[test] + fn snapshot_registration_failure_rolls_back_batch_close_and_cached_head() { + let db = temp_db("snapshot-close-rollback"); + let mut storage = Storage::open(&db.path).unwrap(); + let mut head = storage + .initialize_open_state(10, SafeInputRange::empty_at(0)) + .unwrap(); + let prefix = std::path::Path::new("existing-baseline"); + storage + .insert_baseline_snapshot(prefix, ExecutedInputCount::ZERO) + .unwrap(); + let batch = head.batch_index; + let error = storage + .close_frame_and_batch_with_snapshot( + &mut head, + 10, + prefix, + batch, + ExecutedInputCount::ZERO, + ) + .expect_err("colliding artifact name must roll back the whole close"); + assert!(error.to_string().contains("UNIQUE")); + assert_eq!(head.batch_index, batch); + let persisted = storage.open_state().unwrap().unwrap(); + assert_eq!(persisted.batch_index, batch); + assert_eq!(persisted.frame_in_batch, head.frame_in_batch); + assert_eq!( + storage + .conn + .query_row("SELECT COUNT(*) FROM batches", [], |row| row + .get::<_, i64>(0)) + .unwrap(), + 1 + ); + assert_eq!( + storage + .conn + .query_row("SELECT COUNT(*) FROM valid_closed_batches", [], |row| row + .get::<_, i64>( + 0 + )) + .unwrap(), + 0 + ); + assert_eq!( + storage + .conn + .query_row("SELECT COUNT(*) FROM snapshots", [], |row| row + .get::<_, i64>(0)) + .unwrap(), + 1 + ); + } + + #[test] + #[should_panic(expected = "frame clock cannot regress")] + fn frame_clock_cannot_regress_even_when_the_l1_interval_has_no_inputs() { + let db = temp_db("empty-frame-clock-regression"); + let mut storage = Storage::open(&db.path).unwrap(); + let mut head = storage + .initialize_open_state(10, SafeInputRange::empty_at(0)) + .unwrap(); + storage + .close_frame_only_with_executions(&mut head, 9, SafeInputRange::empty_at(0), &[]) + .unwrap(); + } + #[test] fn open_state_is_idempotent_and_rotation_is_atomic() { let db = temp_db("open-state"); @@ -999,7 +971,7 @@ mod tests { } #[test] - fn mismatched_user_execution_offset_rolls_back_physical_and_logical_rows() { + fn mismatched_execution_offset_rolls_back_source_and_history() { let db = temp_db("user-execution-offset-atomicity"); let mut storage = Storage::open(db.path.as_str()).expect("open storage"); let mut head = storage @@ -1011,13 +983,11 @@ mod tests { .append_executed_user_ops_chunk(&mut head, &[included]) .expect_err("non-canonical execution offset must fail loud"); assert!( - error - .to_string() - .contains("must equal canonical next count"), + error.to_string().contains("must equal next count"), "unexpected trigger error: {error}" ); - for table in ["user_ops", "sequenced_l2_txs", "executed_inputs"] { + for table in ["user_ops", "application_inputs"] { let persisted: i64 = storage .conn .query_row(&format!("SELECT COUNT(*) FROM {table}"), [], |row| { @@ -1032,153 +1002,6 @@ mod tests { ); } - #[test] - fn schema_rejects_execution_attribution_to_a_non_tip_row() { - let db = temp_db("executed-input-non-tip"); - let mut storage = Storage::open(db.path.as_str()).expect("open storage"); - let mut head = storage - .initialize_open_state(0, SafeInputRange::empty_at(0)) - .expect("initialize open state"); - storage - .append_user_ops_chunk(&mut head, &[pending_user_op(0)]) - .expect("append physical-only user op"); - let physical_offset = physical_user_op_offsets(&storage)[0]; - storage - .close_frame_and_batch(&mut head, 0) - .expect("seal the physical row's batch"); - - let err = storage - .conn - .execute( - "INSERT INTO executed_inputs \ - (sequenced_l2_tx_offset, executed_input_offset) VALUES (?1, 0)", - [physical_offset], - ) - .expect_err("a sealed batch cannot acquire execution attribution"); - assert!( - err.to_string().contains("current valid Tip"), - "unexpected trigger error: {err:?}" - ); - } - - #[test] - fn schema_rejects_execution_attribution_before_rebuild_base_is_bound() { - let db = temp_db("executed-input-unbound-base"); - let mut storage = - Storage::initialize_for_command(db.path.as_str(), LifecycleCommand::Rebuild) - .expect("initialize rebuild storage"); - let mut head = storage - .initialize_open_state(0, SafeInputRange::empty_at(0)) - .expect("initialize recovery root fixture"); - storage - .append_user_ops_chunk(&mut head, &[pending_user_op(0)]) - .expect("append physical-only user op"); - let physical_offset = physical_user_op_offsets(&storage)[0]; - - let err = storage - .conn - .execute( - "INSERT INTO executed_inputs \ - (sequenced_l2_tx_offset, executed_input_offset) VALUES (?1, 0)", - [physical_offset], - ) - .expect_err("an unbound rebuild cannot acquire execution attribution"); - assert!( - err.to_string().contains("history base is not bound"), - "unexpected trigger error: {err:?}" - ); - } - - #[test] - fn schema_rejects_execution_attribution_physical_backfill() { - let db = temp_db("executed-input-physical-backfill"); - let mut storage = Storage::open(db.path.as_str()).expect("open storage"); - let mut head = storage - .initialize_open_state(0, SafeInputRange::empty_at(0)) - .expect("initialize open state"); - storage - .append_user_ops_chunk(&mut head, &[pending_user_op(0), pending_user_op(1)]) - .expect("append physical-only user ops"); - let physical_offsets = physical_user_op_offsets(&storage); - storage - .conn - .execute( - "INSERT INTO executed_inputs \ - (sequenced_l2_tx_offset, executed_input_offset) VALUES (?1, 0)", - [physical_offsets[1]], - ) - .expect("seed the later physical attribution"); - - let err = storage - .conn - .execute( - "INSERT INTO executed_inputs \ - (sequenced_l2_tx_offset, executed_input_offset) VALUES (?1, 1)", - [physical_offsets[0]], - ) - .expect_err("execution attribution cannot move backward physically"); - assert!( - err.to_string() - .contains("must follow physical replay order"), - "unexpected trigger error: {err:?}" - ); - } - - #[test] - fn schema_rejects_noncanonical_execution_offset() { - let db = temp_db("executed-input-logical-gap"); - let mut storage = Storage::open(db.path.as_str()).expect("open storage"); - let mut head = storage - .initialize_open_state(0, SafeInputRange::empty_at(0)) - .expect("initialize open state"); - storage - .append_user_ops_chunk(&mut head, &[pending_user_op(0)]) - .expect("append physical-only user op"); - let physical_offset = physical_user_op_offsets(&storage)[0]; - - let err = storage - .conn - .execute( - "INSERT INTO executed_inputs \ - (sequenced_l2_tx_offset, executed_input_offset) VALUES (?1, 1)", - [physical_offset], - ) - .expect_err("the first canonical offset must equal the zero base"); - assert!( - err.to_string().contains("must equal canonical next count"), - "unexpected trigger error: {err:?}" - ); - } - - #[test] - fn schema_rejects_deleting_valid_execution_attribution() { - let db = temp_db("executed-input-valid-delete"); - let mut storage = Storage::open(db.path.as_str()).expect("open storage"); - let mut head = storage - .initialize_open_state(0, SafeInputRange::empty_at(0)) - .expect("initialize open state"); - storage - .append_executed_user_ops_chunk(&mut head, &[included_user_op(0, 0)]) - .expect("append mapped user op"); - - let err = storage - .conn - .execute( - "DELETE FROM executed_inputs WHERE executed_input_offset = 0", - [], - ) - .expect_err("valid canonical attribution is not deletable"); - assert!( - err.to_string() - .contains("valid executed input attribution cannot be deleted"), - "unexpected trigger error: {err:?}" - ); - assert_eq!( - storage.next_executed_input_count().expect("preserved head"), - ExecutedInputCount::new(1) - ); - } - #[test] fn live_direct_rotation_requires_complete_classified_attribution() { let db = temp_db("direct-execution-attribution-complete"); @@ -1213,7 +1036,6 @@ mod tests { 10, SafeInputRange::new(0, 2), &incomplete, - None, ); })); assert!(panic.is_err(), "omitted executable direct must fail loud"); @@ -1235,13 +1057,7 @@ mod tests { }, ]; storage - .close_frame_only_with_executions( - &mut head, - 10, - SafeInputRange::new(0, 2), - &complete, - None, - ) + .close_frame_only_with_executions(&mut head, 10, SafeInputRange::new(0, 2), &complete) .expect("commit complete direct attribution"); assert_eq!( storage.next_executed_input_count().unwrap(), @@ -1294,128 +1110,21 @@ mod tests { let current: Vec = storage .conn - .prepare( - "SELECT executed_input_offset FROM executed_inputs \ - ORDER BY sequenced_l2_tx_offset", - ) + .prepare("SELECT offset FROM application_inputs ORDER BY offset") .unwrap() .query_map([], |row| row.get(0)) .unwrap() .collect::>>() .unwrap(); assert_eq!(current, vec![0, 1]); - let valid: Vec = storage - .conn - .prepare( - "SELECT executed_input_offset FROM valid_executed_inputs \ - ORDER BY sequenced_l2_tx_offset", - ) - .unwrap() - .query_map([], |row| row.get(0)) - .unwrap() - .collect::>>() - .unwrap(); - assert_eq!(valid, vec![0, 1]); - let physical_rows: i64 = storage - .conn - .query_row("SELECT COUNT(*) FROM sequenced_l2_txs", [], |row| { - row.get(0) - }) - .unwrap(); - assert_eq!( - physical_rows, 3, - "invalidation removes only derived mappings, not physical audit rows" - ); - } - - #[test] - fn snapshot_promotion_preserves_executed_input_count() { - let db = temp_db("snapshot-executed-input-count-promotion"); - let mut storage = Storage::open(db.path.as_str()).expect("open storage"); - let mut head = storage - .initialize_open_state(0, SafeInputRange::empty_at(0)) - .expect("initialize open state"); - storage - .append_executed_user_ops_chunk(&mut head, &[included_user_op(0, 0)]) - .expect("append mapped user op"); - let physical_head = storage.valid_ordered_l2_tx_head().unwrap(); - storage - .insert_pending_dump(&db._dir.path().join("pending"), 0, physical_head) - .expect("insert pending checkpoint"); - assert_eq!( - storage - .latest_pending_dump() - .unwrap() - .unwrap() - .executed_input_count, - ExecutedInputCount::new(1) - ); - - storage - .promote_finalized(0, 10) - .expect("promote checkpoint"); - assert_eq!( - storage - .finalized_dump() - .unwrap() - .unwrap() - .executed_input_count, - ExecutedInputCount::new(1), - "promotion must copy the application boundary with the physical cursor" - ); - } - - #[test] - fn cockroach_padding_stays_unmapped_above_absolute_history_base() { - let db = temp_db("cockroach-padding-execution-offsets"); - let mut storage = - Storage::initialize_for_command(db.path.as_str(), LifecycleCommand::Rebuild) - .expect("initialize rebuild storage"); - let recovered = [ - StoredSafeInput { - sender: Address::ZERO, - payload: vec![0xaa], - block_number: 10, - }, - StoredSafeInput { - sender: Address::ZERO, - payload: vec![0xbb], - block_number: 10, - }, - ]; - storage - .append_safe_inputs(10, &recovered, SENDER_A, &default_protocol_timing()) - .expect("persist recovered L1 prefix"); - storage - .open_recovery_tip(10) - .expect("open padded recovery root"); - let physical_head = storage - .valid_ordered_l2_tx_head() - .expect("physical replay head"); - storage - .insert_initial_finalized_dump( - &db._dir.path().join("recovered"), - 10, - physical_head, - 41, - 2, - ) - .expect("bind recovered application base"); - - let mapped: i64 = storage + let source_rows: i64 = storage .conn - .query_row("SELECT COUNT(*) FROM executed_inputs", [], |row| row.get(0)) + .query_row("SELECT COUNT(*) FROM user_ops", [], |row| row.get(0)) .unwrap(); - assert_eq!(mapped, 0, "recovery-root padding is never re-attributed"); assert_eq!( - storage - .next_executed_input_count() - .expect("absolute next count"), - ExecutedInputCount::new(41) + source_rows, 3, + "invalidation preserves original signed operations" ); - let replay = storage.ordered_l2_txs_page_from(0, 10).unwrap(); - assert_eq!(replay.len(), 2); - assert!(replay.iter().all(|row| row.executed_input_offset.is_none())); } #[test] @@ -1551,9 +1260,10 @@ mod tests { } #[test] - fn next_undrained_safe_input_index_is_derived_from_sequenced_directs() { + fn next_undrained_input_is_derived_from_accounted_frame_block() { let db = temp_db("safe-cursor"); let mut storage = Storage::open(db.path.as_str()).expect("open storage"); + pin_deployment_identity(&mut storage, SENDER_A); assert_eq!( storage .next_undrained_safe_input_index() @@ -1618,6 +1328,7 @@ mod tests { fn replay_returns_direct_inputs_in_drain_order() { let db = temp_db("replay-order"); let mut storage = Storage::open(db.path.as_str()).expect("open storage"); + pin_deployment_identity(&mut storage, SENDER_A); let head = storage .initialize_open_state(0, SafeInputRange::empty_at(0)) .expect("initialize open state"); @@ -1642,15 +1353,13 @@ mod tests { .close_frame_only(&mut head, 10, SafeInputRange::new(0, drained.len() as u64)) .expect("close frame with directs"); - let replay = storage - .ordered_l2_txs_page_from(0, 100) - .expect("load replay"); + let replay = crate::storage::test_helpers::all_ordered_l2_txs(&mut storage); assert_eq!(replay.len(), 2); - match &replay[0].tx { + match &replay[0] { SequencedL2Tx::Direct(value) => assert_eq!(value.payload.as_slice(), &[0xaa]), _ => panic!("expected direct input at position 0"), } - match &replay[1].tx { + match &replay[1] { SequencedL2Tx::Direct(value) => assert_eq!(value.payload.as_slice(), &[0xbb]), _ => panic!("expected direct input at position 1"), } @@ -1713,22 +1422,24 @@ mod tests { // The rows are in the ordered L2-tx stream for catch-up to replay and // carry their creation-time application offsets. ensure_open_tip does // not itself run application code; replay validates those offsets. + let bounds = storage.history_bounds().expect("bounds"); let replay = storage - .ordered_l2_txs_page_from(0, 100) - .expect("load replay"); + .canonical_history_page( + sequencer_core::history::HistoryClaim { + version: bounds.version, + next_input: bounds.available_from, + }, + 100, + ) + .expect("load replay") + .rows; assert_eq!( replay.len(), 2, "leading directs are in the replay stream for catch-up" ); - assert_eq!( - replay[0].executed_input_offset, - Some(ExecutedInputCount::ZERO) - ); - assert_eq!( - replay[1].executed_input_offset, - Some(ExecutedInputCount::new(1)) - ); + assert_eq!(replay[0].offset, ExecutedInputCount::ZERO); + assert_eq!(replay[1].offset, ExecutedInputCount::new(1)); } #[test] @@ -1757,4 +1468,74 @@ mod tests { "existing frame's safe_block is preserved, not refreshed" ); } + #[test] + fn application_rows_require_initialized_baseline_and_a_live_tip() { + let db = temp_db("application-history-baseline-required"); + let mut storage = + Storage::initialize_for_command(&db.path, LifecycleCommand::Rebuild).unwrap(); + let mut head = storage + .initialize_open_state(0, SafeInputRange::empty_at(0)) + .unwrap(); + assert!( + storage + .append_executed_user_ops_chunk(&mut head, &[included_user_op(0, 0)]) + .is_err() + ); + storage + .write(|tx| { + super::super::history::initialize_history_in(tx, ExecutedInputCount::ZERO, 0) + }) + .unwrap(); + storage + .append_executed_user_ops_chunk(&mut head, &[included_user_op(0, 0)]) + .unwrap(); + storage.close_frame_and_batch(&mut head, 0).unwrap(); + let err = storage.conn.execute("INSERT INTO application_inputs (offset,batch_index,frame_in_batch,user_op_pos_in_frame) VALUES (1,0,0,0)", []).unwrap_err(); + assert!(err.to_string().contains("current valid Tip")); + assert!( + storage + .conn + .execute("UPDATE application_inputs SET offset = 1", []) + .is_err() + ); + assert!( + storage + .conn + .execute("DELETE FROM application_inputs", []) + .is_err() + ); + } + + #[test] + fn envelopes_advance_accounting_without_becoming_application_inputs() { + let db = temp_db("envelope-only-accounting"); + let mut storage = Storage::open(&db.path).unwrap(); + pin_deployment_identity(&mut storage, SENDER_A); + let mut head = storage + .initialize_open_state(0, SafeInputRange::empty_at(0)) + .unwrap(); + storage + .append_safe_inputs( + 10, + &[StoredSafeInput { + sender: SENDER_A, + payload: vec![0xff], + block_number: 10, + }], + SENDER_A, + &default_protocol_timing(), + ) + .unwrap(); + storage + .close_frame_only_with_executions(&mut head, 10, SafeInputRange::new(0, 1), &[]) + .unwrap(); + assert_eq!(storage.next_undrained_safe_input_index().unwrap(), 1); + assert_eq!( + storage.next_executed_input_count().unwrap(), + ExecutedInputCount::ZERO + ); + assert!(crate::storage::test_helpers::all_ordered_l2_txs(&mut storage).is_empty()); + storage.close_frame_and_batch(&mut head, 10).unwrap(); + assert_eq!(storage.next_undrained_safe_input_index().unwrap(), 1); + } } diff --git a/sequencer/src/storage/l1_submission.rs b/sequencer/src/storage/l1_submission.rs index c588fae0..89e150ed 100644 --- a/sequencer/src/storage/l1_submission.rs +++ b/sequencer/src/storage/l1_submission.rs @@ -148,7 +148,7 @@ impl Storage { CASE WHEN s.user_op_pos_in_frame IS NOT NULL THEN f.fee ELSE NULL END AS fee, CASE WHEN s.safe_input_index IS NOT NULL THEN d.payload ELSE NULL END AS payload, CASE WHEN s.safe_input_index IS NOT NULL THEN d.block_number ELSE NULL END AS block_number - FROM valid_sequenced_l2_txs s + FROM application_inputs s LEFT JOIN user_ops u ON u.batch_index = s.batch_index AND u.frame_in_batch = s.frame_in_batch @@ -999,11 +999,11 @@ mod tests { let mut storage = Storage::open(db.path.as_str()).expect("open storage"); let protocol = default_test_protocol(); - // Local batch 0 (first frame safe_block=100) and batch 1 (first frame + // Local batch 0 (first frame safe_block=1900) and batch 1 (first frame // safe_block=1900). Their landings carry the real wire bytes so the // content-identity check accepts them. let mut head = storage - .initialize_open_state(100, SafeInputRange::empty_at(0)) + .initialize_open_state(1900, SafeInputRange::empty_at(0)) .expect("initialize"); storage .close_frame_and_batch(&mut head, 1900) @@ -1012,7 +1012,7 @@ mod tests { .close_frame_and_batch(&mut head, 1900) .expect("close 1"); - // Non-stale landing of batch 0 (block 200 - safe_block 100 < 1200). + // Non-stale landing of batch 0 (block 1950 - safe_block 1900 < 1200). let non_stale_payload = local_batch_payload(&mut storage, 0); // Stale copy at nonce 1 (safe_block=100, block 2000 → age >= 1200): // a scheduler no-op, so its content is never compared — synthetic @@ -1025,7 +1025,7 @@ mod tests { StoredSafeInput { sender: SENDER_A, payload: non_stale_payload, - block_number: 200, + block_number: 1950, }, StoredSafeInput { sender: SENDER_A, diff --git a/sequencer/src/storage/lifecycle.rs b/sequencer/src/storage/lifecycle.rs index a274e955..3cf089fa 100644 --- a/sequencer/src/storage/lifecycle.rs +++ b/sequencer/src/storage/lifecycle.rs @@ -96,6 +96,41 @@ pub enum LifecycleError { } impl Storage { + /// Publish a durable setup dump and its complete baseline atomically. + pub(crate) fn complete_baseline_setup( + &mut self, + prefix: &std::path::Path, + count: sequencer_core::history::ExecutedInputCount, + safe_block: u64, + next_batch_nonce: u64, + recovery: bool, + ) -> Result<(), LifecycleError> { + let tx = self + .conn + .transaction_with_behavior(TransactionBehavior::Immediate)?; + refuse_on_canonical_divergence(&tx)?; + require_command_fits_completion( + &tx, + if recovery { + LifecycleCommand::Rebuild + } else { + LifecycleCommand::Setup + }, + )?; + super::history::initialize_history_in(&tx, count, safe_block)?; + super::mutations::set_batch_tree_anchor_in(&tx, next_batch_nonce)?; + if recovery { + super::ingress::open_recovery_tip_in_tx(&tx, safe_block)?; + } + super::snapshot_dumps::insert_baseline_snapshot_in(&tx, prefix, count)?; + tx.execute( + "INSERT INTO setup_complete (singleton_id, completed_at_ms) VALUES (0, ?1)", + [now_unix_ms()], + )?; + tx.commit()?; + Ok(()) + } + /// The admission facts, checked read-only before a command does any /// preparatory work: divergence is absorbing, and the two-sided /// completion rule orders commands. @@ -109,28 +144,21 @@ impl Storage { /// Commit setup's timeless completion fact. The `setup_complete` /// primary key makes double-completion unrepresentable at the engine, - /// and the preconditions (finalized snapshot + application-history base) + /// and the preconditions (recovery checkpoint + application-history base) /// are re-read inside the same transaction so completion can never /// outrun the state it certifies. + #[cfg(test)] pub(crate) fn complete_setup(&mut self) -> Result<(), LifecycleError> { let tx = self .conn .transaction_with_behavior(TransactionBehavior::Immediate)?; // Completing setup over persisted divergence would be a lie. refuse_on_canonical_divergence(&tx)?; - let history = super::history::query_history_state(&tx)?; - let has_finalized_snapshot: bool = tx.query_row( - "SELECT EXISTS(SELECT 1 FROM finalized_snapshot WHERE singleton_id = 0)", - [], - |row| row.get(0), - )?; - if history.base_executed_input_count.is_none() - || history.base_safe_input_index.is_none() - || !has_finalized_snapshot - { + super::history::query_history_state(&tx)?; + if !super::snapshot_dumps::has_rollback_safe_snapshot_in(&tx)? { return Err(LifecycleError::Malformed( - "setup cannot complete before its finalized snapshot and \ - application-history base and safe-input floor are established" + "setup cannot complete before its recovery checkpoint and \ + application-history baseline are established" .to_string(), )); } @@ -271,9 +299,14 @@ mod tests { fn complete_seeded_setup(storage: &mut Storage) { storage - .insert_initial_finalized_dump(std::path::Path::new("/tmp/facts-genesis"), 0, 0, 0, 0) - .expect("register finalized snapshot"); - storage.complete_setup().expect("complete setup"); + .complete_baseline_setup( + std::path::Path::new("/tmp/facts-genesis"), + sequencer_core::history::ExecutedInputCount::ZERO, + 0, + 0, + false, + ) + .expect("complete setup"); } fn seed_divergence(storage: &Storage) { @@ -325,9 +358,7 @@ mod tests { #[test] fn divergence_refuses_every_preflight_and_setup_completion() { let (_db, mut storage) = seeded(LifecycleCommand::Setup); - storage - .insert_initial_finalized_dump(std::path::Path::new("/tmp/facts-div"), 0, 0, 0, 0) - .expect("register finalized snapshot"); + seed_divergence(&storage); for command in [ @@ -404,10 +435,7 @@ mod tests { #[test] fn setup_cannot_complete_before_base_and_snapshot_exist() { let (_db, mut storage) = seeded(LifecycleCommand::Rebuild); - assert!(matches!( - storage.complete_setup(), - Err(LifecycleError::Malformed(_)) - )); + assert!(storage.complete_setup().is_err()); assert!(!storage.is_setup_complete().expect("read completion")); } } diff --git a/sequencer/src/storage/migrations/0001_schema.sql b/sequencer/src/storage/migrations/0001_schema.sql index 510dc8b7..e94c8c27 100644 --- a/sequencer/src/storage/migrations/0001_schema.sql +++ b/sequencer/src/storage/migrations/0001_schema.sql @@ -58,9 +58,8 @@ CREATE INDEX IF NOT EXISTS idx_batches_valid_closed_by_nonce WHERE invalidated_at_ms IS NULL AND sealed_at_ms IS NOT NULL; -- ── Views ────────────────────────────────────────────────────────────────── --- Readers over batch data go through the `valid_*` views (here and --- `valid_sequenced_l2_txs` below), which encapsulate the "exclude invalidated --- rows" filter; writers always target the base tables. +-- Readers over batch data go through these `valid_*` views, which exclude +-- invalidated rows; writers always target the base tables. CREATE VIEW IF NOT EXISTS valid_batches AS SELECT * FROM batches WHERE invalidated_at_ms IS NULL; @@ -245,19 +244,6 @@ BEGIN SELECT RAISE(ABORT, 'user_ops can only be inserted into the current Tip'); END; --- Automatically sequence every user-op into the global replay order on insert. --- Note: safe_inputs do NOT have an analogous trigger because their --- batch_index/frame_in_batch are not known at INSERT time — safe inputs --- are ingested by the input reader independently, and only assigned to a --- frame when the frame is closed. The Rust code inserts into --- sequenced_l2_txs explicitly at frame-close time. -CREATE TRIGGER IF NOT EXISTS trg_sequence_user_op AFTER INSERT ON user_ops -BEGIN - INSERT INTO sequenced_l2_txs ( - batch_index, frame_in_batch, user_op_pos_in_frame, safe_input_index - ) VALUES (NEW.batch_index, NEW.frame_in_batch, NEW.pos_in_frame, NULL); -END; - CREATE TABLE IF NOT EXISTS safe_inputs ( safe_input_index INTEGER PRIMARY KEY, sender BLOB NOT NULL CHECK (length(sender) = 20), @@ -274,64 +260,28 @@ CREATE TABLE IF NOT EXISTS safe_inputs ( CREATE INDEX IF NOT EXISTS idx_safe_inputs_sender ON safe_inputs(sender); --- Global append-only replay order consumed by catch-up and feed readers. --- It is a cache, containing the merged and flattened txs of safe_inputs and user_ops. -CREATE TABLE IF NOT EXISTS sequenced_l2_txs ( - offset INTEGER PRIMARY KEY, - batch_index INTEGER NOT NULL, - frame_in_batch INTEGER NOT NULL, - - -- User-op branch: references user_ops(..., pos_in_frame). +-- Current application order. Recovery replaces only an invalidated suffix; +-- immutable source payloads remain in their owning tables. +CREATE TABLE IF NOT EXISTS application_inputs ( + offset INTEGER PRIMARY KEY CHECK (typeof(offset) = 'integer' AND offset >= 0), + batch_index INTEGER NOT NULL, + frame_in_batch INTEGER NOT NULL, user_op_pos_in_frame INTEGER, - - -- Direct-input branch: references safe_inputs(safe_input_index). - safe_input_index INTEGER, - + safe_input_index INTEGER, FOREIGN KEY(batch_index, frame_in_batch) REFERENCES frames(batch_index, frame_in_batch), FOREIGN KEY(batch_index, frame_in_batch, user_op_pos_in_frame) REFERENCES user_ops(batch_index, frame_in_batch, pos_in_frame), - FOREIGN KEY(safe_input_index) - REFERENCES safe_inputs(safe_input_index), - - -- XOR invariant: row is either a sequenced user-op OR a drained direct input. - CHECK ( - (user_op_pos_in_frame IS NOT NULL AND safe_input_index IS NULL) OR - (user_op_pos_in_frame IS NULL AND safe_input_index IS NOT NULL) - ), - - -- At most one sequenced user-op row for each user-op key. - UNIQUE(batch_index, frame_in_batch, user_op_pos_in_frame) - -- A direct input may be sequenced more than once if its original batch is - -- invalidated and a recovery batch re-drains it. The read-side query filters - -- out rows from invalid batches, so only the latest valid drain is visible. - -- (No UNIQUE constraint on safe_input_index.) + FOREIGN KEY(safe_input_index) REFERENCES safe_inputs(safe_input_index), + CHECK ((user_op_pos_in_frame IS NOT NULL AND safe_input_index IS NULL) + OR (user_op_pos_in_frame IS NULL AND safe_input_index IS NOT NULL)), + UNIQUE(batch_index, frame_in_batch, user_op_pos_in_frame), + UNIQUE(safe_input_index) ); - -CREATE TRIGGER IF NOT EXISTS trg_sequenced_l2_txs_target_must_be_tip -BEFORE INSERT ON sequenced_l2_txs -FOR EACH ROW -WHEN NOT EXISTS ( - SELECT 1 FROM batches - WHERE batch_index = NEW.batch_index - AND sealed_at_ms IS NULL - AND invalidated_at_ms IS NULL -) -BEGIN - SELECT RAISE(ABORT, 'sequenced_l2_txs can only target the current Tip'); -END; - -CREATE INDEX IF NOT EXISTS idx_sequenced_l2_txs_frame - ON sequenced_l2_txs(batch_index, frame_in_batch); - --- Partial index for efficient MAX(safe_input_index) lookups used to compute --- the next undrained direct-input cursor at frame-close time. -CREATE INDEX IF NOT EXISTS idx_sequenced_l2_txs_safe_input - ON sequenced_l2_txs(safe_input_index) WHERE safe_input_index IS NOT NULL; - -CREATE VIEW IF NOT EXISTS valid_sequenced_l2_txs AS -SELECT * FROM sequenced_l2_txs -WHERE batch_index NOT IN (SELECT batch_index FROM batches WHERE invalidated_at_ms IS NOT NULL); +CREATE INDEX IF NOT EXISTS idx_application_inputs_frame + ON application_inputs(batch_index, frame_in_batch); +CREATE INDEX IF NOT EXISTS idx_safe_inputs_block + ON safe_inputs(block_number, safe_input_index); -- Derived log of batch submissions the scheduler would actually execute. -- Unlike a raw log of all safe submissions, this only contains the accepted @@ -392,7 +342,7 @@ CREATE TABLE IF NOT EXISTS canonical_divergence ( ); -- I15 structural enforcement: while the divergence marker exists, the batch --- tree, promotions, and the pending-snapshot pool are frozen in the engine +-- tree and snapshot collection are frozen in the engine -- itself. Standard recovery is forbidden on a diverged frontier; the typed -- Rust refusals (the local-first startup recovery procedure plus guarded Tip/Cascade -- mutations and atomic runtime admission) remain the friendly error surface, but these @@ -409,213 +359,67 @@ BEFORE UPDATE ON batches FOR EACH ROW WHEN EXISTS (SELECT 1 FROM canonical_divergence WHERE singleton_id = 0) BEGIN SELECT RAISE(ABORT, 'batch tree frozen: canonical divergence marker present'); END; --- External history identity. One database serves exactly one era. The era is --- minted with the baseline schema; standard recovery advances only the --- generation. A rebuild's application-history base and durable safe-input --- drain floor are unknown until the recovered finalized snapshot exists, so --- they alone start NULL and fill together exactly once before setup completes. +-- A complete immutable baseline is registered with its durable artifact. +-- Only the recovery generation changes during this database's lifetime. CREATE TABLE IF NOT EXISTS history_state ( - singleton_id INTEGER PRIMARY KEY CHECK (singleton_id = 0), - era_id BLOB NOT NULL CHECK ( - typeof(era_id) = 'blob' - AND length(era_id) = 16 + singleton_id INTEGER PRIMARY KEY CHECK (singleton_id = 0), + era_id BLOB NOT NULL CHECK ( + typeof(era_id) = 'blob' AND length(era_id) = 16 AND substr(hex(era_id), 13, 1) = '4' - AND substr(hex(era_id), 17, 1) IN ('8', '9', 'A', 'B') - ), - era_created_at_ms INTEGER NOT NULL CHECK ( - typeof(era_created_at_ms) = 'integer' - AND era_created_at_ms >= 0 - ), - recovery_generation INTEGER NOT NULL CHECK ( - typeof(recovery_generation) = 'integer' - AND recovery_generation >= 0 - ), - base_executed_input_count INTEGER CHECK ( - base_executed_input_count IS NULL - OR ( - typeof(base_executed_input_count) = 'integer' - AND base_executed_input_count >= 0 - ) - ), - base_safe_input_index INTEGER CHECK ( - base_safe_input_index IS NULL - OR ( - typeof(base_safe_input_index) = 'integer' - AND base_safe_input_index >= 0 - ) - ), - CHECK ( - (base_executed_input_count IS NULL AND base_safe_input_index IS NULL) - OR - (base_executed_input_count IS NOT NULL AND base_safe_input_index IS NOT NULL) - ) + AND substr(hex(era_id), 17, 1) IN ('8', '9', 'A', 'B')), + era_created_at_ms INTEGER NOT NULL CHECK ( + typeof(era_created_at_ms) = 'integer' AND era_created_at_ms >= 0), + recovery_generation INTEGER NOT NULL CHECK ( + typeof(recovery_generation) = 'integer' AND recovery_generation >= 0), + base_executed_input_count INTEGER NOT NULL CHECK ( + typeof(base_executed_input_count) = 'integer' AND base_executed_input_count >= 0), + base_safe_block INTEGER NOT NULL CHECK ( + typeof(base_safe_block) = 'integer' AND base_safe_block >= 0) ); - CREATE TRIGGER IF NOT EXISTS trg_history_state_single_insert -BEFORE INSERT ON history_state -FOR EACH ROW -WHEN EXISTS (SELECT 1 FROM history_state WHERE singleton_id = 0) -BEGIN - SELECT RAISE(ABORT, 'history state is inserted once per database'); -END; - +BEFORE INSERT ON history_state FOR EACH ROW +WHEN EXISTS (SELECT 1 FROM history_state) +BEGIN SELECT RAISE(ABORT, 'history state is inserted once per database'); END; CREATE TRIGGER IF NOT EXISTS trg_history_identity_write_once -BEFORE UPDATE OF singleton_id, era_id, era_created_at_ms ON history_state -FOR EACH ROW -BEGIN - SELECT RAISE(ABORT, 'history era identity is write-once'); -END; - -CREATE TRIGGER IF NOT EXISTS trg_history_base_write_once -BEFORE UPDATE OF base_executed_input_count, base_safe_input_index ON history_state -FOR EACH ROW -WHEN OLD.base_executed_input_count IS NOT NULL - OR OLD.base_safe_input_index IS NOT NULL -BEGIN - SELECT RAISE(ABORT, 'history base is write-once'); -END; - +BEFORE UPDATE OF singleton_id, era_id, era_created_at_ms, + base_executed_input_count, base_safe_block ON history_state FOR EACH ROW +BEGIN SELECT RAISE(ABORT, 'history baseline is immutable'); END; CREATE TRIGGER IF NOT EXISTS trg_history_generation_monotonic -BEFORE UPDATE OF recovery_generation ON history_state -FOR EACH ROW +BEFORE UPDATE OF recovery_generation ON history_state FOR EACH ROW WHEN OLD.recovery_generation = 9223372036854775807 OR NEW.recovery_generation != OLD.recovery_generation + 1 -BEGIN - SELECT RAISE(ABORT, 'recovery generation must advance by exactly one'); -END; - +BEGIN SELECT RAISE(ABORT, 'recovery generation must advance by exactly one'); END; CREATE TRIGGER IF NOT EXISTS trg_history_state_not_deletable -BEFORE DELETE ON history_state -FOR EACH ROW -BEGIN - SELECT RAISE(ABORT, 'history state is write-once per database'); -END; - --- Canonical application-history coordinates attached to physical replay rows. --- --- `sequenced_l2_txs.offset` remains the append-only SQLite pagination cursor: --- it may contain invalidated rows and rows that the application never executes --- (our own batch submissions and cockroach-root cursor padding). This table is --- the separate, sparse attribution saying which physical rows did execute and --- at which `Application::executed_input_count` boundary. --- --- The primary key makes attribution one-to-one per physical row. This is a --- derived *current canonical projection*, not the audit log: invalidating a --- batch atomically deletes its mappings while retaining the physical replay --- rows. The replacement suffix can then reuse its canonical offsets, enforced --- by the global logical UNIQUE constraint. -CREATE TABLE IF NOT EXISTS executed_inputs ( - sequenced_l2_tx_offset INTEGER PRIMARY KEY - REFERENCES sequenced_l2_txs(offset), - executed_input_offset INTEGER NOT NULL CHECK ( - typeof(executed_input_offset) = 'integer' - AND executed_input_offset >= 0 - ), - UNIQUE(executed_input_offset) -); - --- Invalidation structurally deletes mappings below, so this projection can --- join the physical table directly without re-running the valid-batch filter. -CREATE VIEW IF NOT EXISTS valid_executed_inputs AS -SELECT - e.sequenced_l2_tx_offset, - e.executed_input_offset, - s.batch_index, - s.frame_in_batch, - s.user_op_pos_in_frame, - s.safe_input_index -FROM executed_inputs e -JOIN sequenced_l2_txs s ON s.offset = e.sequenced_l2_tx_offset; - --- Attribution is creation-time state, not a catch-up repair operation. The --- Rust writer maps rows in their creation transaction; this backstop limits a --- target to the current valid Tip and refuses physical-order rewrites. -CREATE TRIGGER IF NOT EXISTS trg_executed_inputs_target_must_be_tip -BEFORE INSERT ON executed_inputs -FOR EACH ROW -WHEN NOT EXISTS ( - SELECT 1 - FROM sequenced_l2_txs s - JOIN batches b ON b.batch_index = s.batch_index - WHERE s.offset = NEW.sequenced_l2_tx_offset - AND b.sealed_at_ms IS NULL - AND b.invalidated_at_ms IS NULL -) -BEGIN - SELECT RAISE(ABORT, 'executed input must target the current valid Tip'); -END; - -CREATE TRIGGER IF NOT EXISTS trg_executed_inputs_requires_bound_base -BEFORE INSERT ON executed_inputs -FOR EACH ROW -WHEN (SELECT base_executed_input_count FROM history_state WHERE singleton_id = 0) IS NULL -BEGIN - SELECT RAISE(ABORT, 'executed input history base is not bound'); -END; - -CREATE TRIGGER IF NOT EXISTS trg_executed_inputs_physical_order -BEFORE INSERT ON executed_inputs -FOR EACH ROW -WHEN EXISTS ( - SELECT 1 FROM executed_inputs - WHERE sequenced_l2_tx_offset >= NEW.sequenced_l2_tx_offset -) -BEGIN - SELECT RAISE(ABORT, 'executed input attributions must follow physical replay order'); -END; - --- Every new mapping consumes exactly the current canonical next offset: --- max(the era base, MAX(current mapping) + 1). Invalidation deletes its suffix --- mappings, naturally rewinding the next offset for the replacement suffix. -CREATE TRIGGER IF NOT EXISTS trg_executed_inputs_contiguous -BEFORE INSERT ON executed_inputs -FOR EACH ROW -WHEN NEW.executed_input_offset != ( - SELECT MAX( - base_executed_input_count, - COALESCE((SELECT MAX(executed_input_offset) + 1 FROM executed_inputs), 0) - ) - FROM history_state - WHERE singleton_id = 0 -) -BEGIN - SELECT RAISE(ABORT, 'executed input offset must equal canonical next count'); -END; - -CREATE TRIGGER IF NOT EXISTS trg_executed_inputs_append_only_update -BEFORE UPDATE ON executed_inputs -FOR EACH ROW -BEGIN - SELECT RAISE(ABORT, 'executed input attribution is append-only'); -END; - -CREATE TRIGGER IF NOT EXISTS trg_protect_valid_executed_input_delete -BEFORE DELETE ON executed_inputs -FOR EACH ROW -WHEN EXISTS ( - SELECT 1 - FROM sequenced_l2_txs s - JOIN batches b ON b.batch_index = s.batch_index - WHERE s.offset = OLD.sequenced_l2_tx_offset - AND b.invalidated_at_ms IS NULL -) -BEGIN - SELECT RAISE(ABORT, 'valid executed input attribution cannot be deleted'); -END; - --- Recovery owns the only deletion path. The batch row is already invalid when --- this AFTER trigger runs, so the guarded delete above permits exactly these --- derived mappings to disappear in the same transaction as suffix invalidation. -CREATE TRIGGER IF NOT EXISTS trg_drop_invalidated_executed_inputs -AFTER UPDATE OF invalidated_at_ms ON batches -FOR EACH ROW +BEFORE DELETE ON history_state FOR EACH ROW +BEGIN SELECT RAISE(ABORT, 'history state is write-once per database'); END; + +CREATE TRIGGER IF NOT EXISTS trg_application_inputs_target_must_be_tip +BEFORE INSERT ON application_inputs FOR EACH ROW +WHEN NOT EXISTS (SELECT 1 FROM batches WHERE batch_index = NEW.batch_index + AND sealed_at_ms IS NULL AND invalidated_at_ms IS NULL) +BEGIN SELECT RAISE(ABORT, 'application input must target the current valid Tip'); END; +CREATE TRIGGER IF NOT EXISTS trg_application_inputs_requires_baseline +BEFORE INSERT ON application_inputs FOR EACH ROW +WHEN NOT EXISTS (SELECT 1 FROM history_state) +BEGIN SELECT RAISE(ABORT, 'application history baseline is missing'); END; +CREATE TRIGGER IF NOT EXISTS trg_application_inputs_contiguous +BEFORE INSERT ON application_inputs FOR EACH ROW +WHEN NEW.offset != (SELECT MAX(base_executed_input_count, + COALESCE((SELECT MAX(offset) + 1 FROM application_inputs), 0)) + FROM history_state WHERE singleton_id = 0) +BEGIN SELECT RAISE(ABORT, 'application input offset must equal next count'); END; +CREATE TRIGGER IF NOT EXISTS trg_application_inputs_immutable +BEFORE UPDATE ON application_inputs FOR EACH ROW +BEGIN SELECT RAISE(ABORT, 'application inputs are immutable'); END; +CREATE TRIGGER IF NOT EXISTS trg_protect_valid_application_input_delete +BEFORE DELETE ON application_inputs FOR EACH ROW +WHEN EXISTS (SELECT 1 FROM batches WHERE batch_index = OLD.batch_index + AND invalidated_at_ms IS NULL) +BEGIN SELECT RAISE(ABORT, 'valid application input cannot be deleted'); END; +CREATE TRIGGER IF NOT EXISTS trg_drop_invalidated_application_inputs +AFTER UPDATE OF invalidated_at_ms ON batches FOR EACH ROW WHEN OLD.invalidated_at_ms IS NULL AND NEW.invalidated_at_ms IS NOT NULL -BEGIN - DELETE FROM executed_inputs - WHERE sequenced_l2_tx_offset IN ( - SELECT offset FROM sequenced_l2_txs WHERE batch_index = NEW.batch_index - ); -END; +BEGIN DELETE FROM application_inputs WHERE batch_index = NEW.batch_index; END; -- Terminal-fault black box: an append-only trail of terminal causes, -- best-effort recorded before death. DELIBERATELY NOT AN ADMISSION GATE: @@ -683,14 +487,9 @@ CREATE TABLE IF NOT EXISTS deployment_identity ( ) ); --- setup-complete marker. The `setup` subcommand --- pins deployment identity, does the initial L1 sync, and registers the --- genesis finalized snapshot; it inserts this singleton row as its LAST --- write. `run` refuses to boot unless the row is present. Presence is the --- single linearization point for "setup finished": it distinguishes a clean --- setup from one that crashed midway (identity pinned and/or genesis --- snapshot registered, but the marker absent), which every prior setup step --- is individually idempotent enough to let `setup` re-run and complete. +-- Setup publishes the complete baseline, snapshot, anchor, and this marker in +-- one final transaction after artifact durability. Earlier identity/sync work +-- can be retried; run cannot admit an incomplete setup. CREATE TABLE IF NOT EXISTS setup_complete ( singleton_id INTEGER PRIMARY KEY CHECK (singleton_id = 0), completed_at_ms INTEGER NOT NULL CHECK (completed_at_ms >= 0) @@ -805,38 +604,10 @@ FROM batch_policy; -- --------------------------------------------------------------------------- -- Snapshot dumps -- --- Three tables together implement the snapshot lifecycle: --- --- * dumps — master table: every on-disk dump has a row here, --- with a lease_count tracking in-flight readers --- (typically HTTP handlers streaming the dump). --- Rows are FK-referenced by pending_snapshots and --- finalized_snapshot via ON DELETE RESTRICT, so a --- dump can't be removed while still referenced. --- * pending_snapshots — one row per batch that has been closed and --- dumped, but not yet observed landed on L1. Keyed --- by nonce so the inclusion lane can match its own --- batches in the direct-input stream. --- * finalized_snapshot — single-row table holding the latest L1-finalized --- snapshot. INSERT OR REPLACE on promotion; --- consumers (the watchdog) read this row to learn --- which dump corresponds to the canonical state. --- --- Garbage collection: dumps with lease_count = 0 AND no row in either --- pending_snapshots or finalized_snapshot are eligible for filesystem + --- DB-row removal. The Rust caller drives this; the FK constraints prevent --- accidental deletion while a reference still exists. --- --- Lifecycle: --- * batch close: INSERT into dumps; INSERT into pending_snapshots. --- * batch observed: INSERT OR REPLACE into finalized_snapshot; DELETE --- the promoted nonces from pending_snapshots in one tx. --- The previous finalized's dump becomes GC-eligible. --- * cascade invalidate: DELETE from pending_snapshots; sweep dumps via GC. --- * HTTP serving: acquire/release lease_count to prevent GC during --- in-flight streams. --- * startup: UPDATE dumps SET lease_count = 0 (clear stale --- in-process leases from a crashed previous run). +-- Immutable snapshots belong to one local batch, or to the era baseline. +-- Acceptance is derived from safe_accepted_batches; it never mutates snapshots. +-- Lease acquisition and garbage collection serialize in SQLite. Files are +-- durable before insertion; database rows are deleted before filesystem cleanup. -- --------------------------------------------------------------------------- CREATE TABLE IF NOT EXISTS dumps ( @@ -846,39 +617,21 @@ CREATE TABLE IF NOT EXISTS dumps ( CHECK (typeof(lease_count) = 'integer' AND lease_count >= 0) ); -CREATE TABLE IF NOT EXISTS pending_snapshots ( - nonce INTEGER PRIMARY KEY CHECK (typeof(nonce) = 'integer' AND nonce >= 0), - dump_id INTEGER NOT NULL REFERENCES dumps(id) ON DELETE RESTRICT, - l2_tx_index INTEGER NOT NULL - CHECK (typeof(l2_tx_index) = 'integer' AND l2_tx_index >= 0), - executed_input_count INTEGER NOT NULL CHECK ( - typeof(executed_input_count) = 'integer' - AND executed_input_count >= 0 - ) -); - -CREATE TABLE IF NOT EXISTS finalized_snapshot ( - singleton_id INTEGER PRIMARY KEY CHECK (singleton_id = 0), - dump_id INTEGER NOT NULL REFERENCES dumps(id) ON DELETE RESTRICT, - inclusion_block INTEGER NOT NULL - CHECK (typeof(inclusion_block) = 'integer' AND inclusion_block >= 0), - l2_tx_index INTEGER NOT NULL - CHECK (typeof(l2_tx_index) = 'integer' AND l2_tx_index >= 0), +CREATE TABLE IF NOT EXISTS snapshots ( + dump_id INTEGER PRIMARY KEY REFERENCES dumps(id) ON DELETE RESTRICT, + batch_index INTEGER UNIQUE REFERENCES batches(batch_index) ON DELETE RESTRICT, executed_input_count INTEGER NOT NULL CHECK ( - typeof(executed_input_count) = 'integer' - AND executed_input_count >= 0 + typeof(executed_input_count) = 'integer' AND executed_input_count >= 0 ) ); +CREATE UNIQUE INDEX IF NOT EXISTS snapshots_single_baseline +ON snapshots ((1)) WHERE batch_index IS NULL; --- I15 structural enforcement, snapshot half (batch-tree half lives next to --- the canonical_divergence table): promotions and pending-pool clears are --- frozen while the divergence marker exists. -CREATE TRIGGER IF NOT EXISTS trg_promotion_frozen_on_divergence -BEFORE INSERT ON finalized_snapshot FOR EACH ROW -WHEN EXISTS (SELECT 1 FROM canonical_divergence WHERE singleton_id = 0) -BEGIN SELECT RAISE(ABORT, 'promotion frozen: canonical divergence marker present'); END; +CREATE TRIGGER IF NOT EXISTS trg_snapshot_immutable +BEFORE UPDATE ON snapshots +BEGIN SELECT RAISE(ABORT, 'snapshot boundaries are immutable'); END; -CREATE TRIGGER IF NOT EXISTS trg_pending_clear_frozen_on_divergence -BEFORE DELETE ON pending_snapshots FOR EACH ROW +CREATE TRIGGER IF NOT EXISTS trg_snapshot_clear_frozen_on_divergence +BEFORE DELETE ON snapshots FOR EACH ROW WHEN EXISTS (SELECT 1 FROM canonical_divergence WHERE singleton_id = 0) -BEGIN SELECT RAISE(ABORT, 'pending-snapshot clear frozen: canonical divergence marker present'); END; +BEGIN SELECT RAISE(ABORT, 'snapshot collection frozen: canonical divergence marker present'); END; diff --git a/sequencer/src/storage/mod.rs b/sequencer/src/storage/mod.rs index 3d8db2d9..9b1308dc 100644 --- a/sequencer/src/storage/mod.rs +++ b/sequencer/src/storage/mod.rs @@ -15,7 +15,7 @@ //! - `recovery` — cascade invalidation, recovery-batch open, danger checks //! - `admin` — operator policy alpha tuning //! - `fee_oracle` — L1 fee-oracle gas-price updates -//! - `snapshot_dumps` — pending/finalized snapshot lifecycle, lease counts +//! - `snapshot_dumps` — immutable snapshots, derived acceptance, lease counts //! - `history` — write-once era/base metadata and recovery generation //! - `lifecycle` — command-admission facts + the terminal-fault black box //! @@ -52,7 +52,9 @@ pub(crate) use convert::is_persistent_storage_error; use std::time::SystemTime; use thiserror::Error; -pub(crate) use egress::L2TxContext; +#[cfg(test)] +pub(crate) use egress::ApplicationInputRow; +pub(crate) use egress::{HistoryReadError, L2TxContext}; pub use history::{DirectInputExecution, HistoryState}; pub use lifecycle::{LifecycleCommand, LifecycleError, TerminalFault}; pub use open::Storage; @@ -60,8 +62,8 @@ pub use recovery::DangerStatus; pub(crate) use recovery::{RecoveryInspection, RecoveryMutationError}; pub use sequencer_core::history::{EraId, ExecutedInputCount, HistoryVersion, RecoveryGeneration}; pub use snapshot_dumps::{ - DumpRow, FinalizedDump, FinalizedLease, LeaseGuard, LeasedDump, PendingDump, - PersistentReleaseFailureReporter, ReleaseScheduler, + DumpRow, FinalizedDump, FinalizedLease, LeaseGuard, LeasedDump, + PersistentReleaseFailureReporter, ReleaseScheduler, Snapshot, }; /// One safe input as stored on the L1 InputBox: sender, opaque payload, and @@ -74,6 +76,13 @@ pub struct StoredSafeInput { pub block_number: u64, } +/// A sender-classified L1 direct input ready for application execution. +#[derive(Debug, Clone)] +pub(crate) struct StoredDirectInput { + pub safe_input_index: u64, + pub input: sequencer_core::l2_tx::DirectInput, +} + /// One InputBox event with the L1 provenance persisted for feed consumers. #[derive(Debug, Clone, PartialEq, Eq)] pub(crate) struct IngestedSafeInput { diff --git a/sequencer/src/storage/mutations.rs b/sequencer/src/storage/mutations.rs index 04e90ca6..e62cf6f4 100644 --- a/sequencer/src/storage/mutations.rs +++ b/sequencer/src/storage/mutations.rs @@ -11,9 +11,7 @@ use alloy_primitives::Address; use rusqlite::{Connection, Result, Transaction, params}; use super::convert::{i64_to_u64, u64_to_i64}; -use super::history::{ - ExecutedInputMapping, attach_executed_inputs_in, next_executed_input_count_in, -}; +use super::history::next_executed_input_count_in; use super::l1_inputs::query_deployment_identity; use super::{DirectInputExecution, SafeInputRange}; @@ -162,9 +160,7 @@ pub(super) fn batch_tree_anchor_in(conn: &Connection) -> Result { Ok(i64_to_u64(anchor)) } -/// Insert one `sequenced_l2_txs` row per safe-input index in `range` for the -/// given (batch, frame). Used by ingress (frame close) and recovery (re-drain -/// after cascade invalidation). +/// Commit the exact direct inputs executed while opening a frame. pub(super) fn persist_frame_direct_sequence( tx: &Transaction<'_>, batch_index: u64, @@ -172,17 +168,16 @@ pub(super) fn persist_frame_direct_sequence( range: SafeInputRange, executions: &[DirectInputExecution], ) -> Result<()> { - let expected = derive_direct_input_executions_in(tx, range)?; assert_eq!( - executions, expected, - "direct execution attributions must cover exactly the non-submitter drained rows from the canonical next count" + executions, + derive_direct_input_executions_in(tx, range)?, + "direct executions must cover the complete external-input range at the next application count" ); - persist_frame_direct_sequence_inner(tx, batch_index, frame_in_batch, range, executions) + persist_direct_executions(tx, batch_index, frame_in_batch, executions) } -/// Sequence a production startup/recovery Tip's leading direct range, deriving -/// its complete application attribution from the persisted deployment identity -/// and current canonical application boundary. +/// Startup fixes the replay sequence before the engine is restored. Admission +/// succeeds only after catch-up applies every committed application input. pub(super) fn persist_frame_direct_sequence_derived( tx: &Transaction<'_>, batch_index: u64, @@ -190,81 +185,27 @@ pub(super) fn persist_frame_direct_sequence_derived( range: SafeInputRange, ) -> Result<()> { let executions = derive_direct_input_executions_in(tx, range)?; - persist_frame_direct_sequence_inner(tx, batch_index, frame_in_batch, range, &executions) + persist_direct_executions(tx, batch_index, frame_in_batch, &executions) } -/// Sequence physical cursor rows that intentionally represent no newly -/// executed application inputs. Cockroach-root padding uses this because the -/// recovered snapshot already contains their effects; test fixtures use it -/// when exercising only the physical ordering layer. -pub(super) fn persist_frame_direct_sequence_physical_only( +fn persist_direct_executions( tx: &Transaction<'_>, batch_index: u64, frame_in_batch: u32, - range: SafeInputRange, -) -> Result<()> { - persist_frame_direct_sequence_inner(tx, batch_index, frame_in_batch, range, &[]) -} - -fn persist_frame_direct_sequence_inner( - tx: &Transaction<'_>, - batch_index: u64, - frame_in_batch: u32, - range: SafeInputRange, executions: &[DirectInputExecution], ) -> Result<()> { - if range.is_empty() { - assert!( - executions.is_empty(), - "empty safe-input range cannot carry execution attributions" - ); - return Ok(()); - } - - let mut previous_safe_input_index = None; - for execution in executions { - assert!( - execution.safe_input_index >= range.start() && execution.safe_input_index < range.end(), - "direct execution attribution lies outside its drained range" - ); - if let Some(previous) = previous_safe_input_index { - assert!( - execution.safe_input_index > previous, - "direct execution attributions must be strictly ordered by safe-input index" - ); - } - previous_safe_input_index = Some(execution.safe_input_index); - } - let mut stmt = tx.prepare_cached( - "INSERT INTO sequenced_l2_txs (batch_index, frame_in_batch, user_op_pos_in_frame, safe_input_index) \ - VALUES (?1, ?2, NULL, ?3)", + "INSERT INTO application_inputs (offset, batch_index, frame_in_batch, safe_input_index) + VALUES (?1, ?2, ?3, ?4)", )?; - let mut executions = executions.iter().peekable(); - let mut mappings = Vec::with_capacity(executions.len()); - for safe_input_index in range.start()..range.end() { + for execution in executions { stmt.execute(params![ + u64_to_i64(execution.executed_input_offset.get()), u64_to_i64(batch_index), i64::from(frame_in_batch), - u64_to_i64(safe_input_index), + u64_to_i64(execution.safe_input_index) ])?; - if executions - .peek() - .is_some_and(|execution| execution.safe_input_index == safe_input_index) - { - let execution = executions.next().expect("peeked direct execution"); - mappings.push(ExecutedInputMapping { - sequenced_l2_tx_offset: i64_to_u64(tx.last_insert_rowid()), - executed_input_offset: execution.executed_input_offset, - }); - } } - assert!( - executions.next().is_none(), - "not every direct execution attribution was persisted" - ); - drop(stmt); - attach_executed_inputs_in(tx, &mappings)?; Ok(()) } diff --git a/sequencer/src/storage/open.rs b/sequencer/src/storage/open.rs index b7579f88..89c7d895 100644 --- a/sequencer/src/storage/open.rs +++ b/sequencer/src/storage/open.rs @@ -6,10 +6,10 @@ //! Method clusters live in sibling files (`ingress`, `egress`, `l1_inputs`, //! `l1_submission`, `recovery`, `admin`) — each adds its own `impl Storage`. -use rusqlite::{Connection, OpenFlags, Result, Transaction, TransactionBehavior, types::Type}; +use rusqlite::{Connection, OpenFlags, Result, Transaction, TransactionBehavior}; use rusqlite_migration::{HookResult, M, Migrations}; -use super::{EraId, LifecycleCommand, StorageOpenError}; +use super::{LifecycleCommand, StorageOpenError}; const MIGRATION_0001_SCHEMA: &str = include_str!("migrations/0001_schema.sql"); @@ -23,14 +23,8 @@ const MIGRATION_0001_SCHEMA: &str = include_str!("migrations/0001_schema.sql"); /// scheduler executed. The dump side already pays the same cost /// (`create_dump` fsyncs); this closes the DB half. Also a precondition /// for the wallet-nonce watermark's write-before-broadcast guarantee. -/// And it is what makes the setup completion transaction a valid -/// linearization point: it commits after the genesis-snapshot row's -/// transaction, so "completion durable ⇒ -/// snapshot row durable ⇒ dump dir durable" only holds because FULL fsyncs -/// every commit — under NORMAL the completion WAL frame could survive while -/// the snapshot row's frames are lost, and `run` would boot a half-set-up -/// DB. Benchmarked at the flip: round-trip/ack deltas were noise-level on -/// NVMe. +/// Setup publishes its complete baseline and completion together after the +/// artifact is durable; FULL makes that boundary survive power loss too. /// /// Do not relax to NORMAL without revisiting all three (externalized /// commits, the write-before-broadcast watermark, and the setup-completion @@ -58,7 +52,7 @@ impl Storage { /// database is created by an owning command through /// [`Storage::initialize_for_command`], so a missing file here is a /// deployment mistake (mistyped `--data-dir`, wrong mount). Creating one - /// on the fly would mint an ownerless era with no creating command — + /// on the fly would create an ownerless schema with no creating command — /// database absence means uninitialized, never create-and-proceed. /// Crate tests keep create-on-open as their fixture idiom; the /// command-less baseline in [`baseline_migration`] exists for them. @@ -77,8 +71,8 @@ impl Storage { }) } - /// Create the baseline schema and history era in one migration - /// transaction, with the creating command deciding the history bases. On + /// Create the schema and record its owning command in one migration + /// transaction. The complete history baseline is published later. On /// an already-migrated database the hook does not run; callers must /// inspect the existing facts. pub(crate) fn initialize_for_command( @@ -211,27 +205,13 @@ fn baseline_migration( post_initial_metadata: Option, ) -> M<'static> { M::up_with_hook(MIGRATION_0001_SCHEMA, move |tx: &Transaction<'_>| { - let recorded_at_ms = i64::try_from(crate::clock::unix_now_ms()).unwrap_or(i64::MAX); - let era_id = mint_era_id(tx)?; - let (base_executed_input_count, base_safe_input_index) = match initial_command { - Some(LifecycleCommand::Rebuild) => (None, None), - Some(LifecycleCommand::Setup) | None => (Some(0_i64), Some(0_i64)), - Some(LifecycleCommand::Run | LifecycleCommand::MaintenanceFlush) => { - unreachable!("baseline command was checked before migration") - } - }; - tx.execute( - "INSERT INTO history_state \ - (singleton_id, era_id, era_created_at_ms, recovery_generation, \ - base_executed_input_count, base_safe_input_index) \ - VALUES (0, ?1, ?2, 0, ?3, ?4)", - rusqlite::params![ - era_id.as_bytes().as_slice(), - recorded_at_ms, - base_executed_input_count, - base_safe_input_index - ], - )?; + if initial_command.is_none() { + super::history::initialize_history_in( + tx, + sequencer_core::history::ExecutedInputCount::ZERO, + 0, + )?; + } if let Some(hook) = post_initial_metadata { hook(tx)?; } @@ -239,28 +219,6 @@ fn baseline_migration( }) } -fn mint_era_id(tx: &Transaction<'_>) -> Result { - let random = tx.query_row("SELECT randomblob(16)", [], |row| row.get::<_, Vec>(0))?; - let mut bytes: [u8; EraId::BYTE_LEN] = random.try_into().map_err(|value: Vec| { - rusqlite::Error::FromSqlConversionFailure( - 0, - Type::Blob, - Box::new(std::io::Error::new( - std::io::ErrorKind::InvalidData, - format!( - "SQLite randomblob returned {} bytes, expected {}", - value.len(), - EraId::BYTE_LEN - ), - )), - ) - })?; - bytes[6] = (bytes[6] & 0x0f) | 0x40; - bytes[8] = (bytes[8] & 0x3f) | 0x80; - EraId::from_bytes(bytes) - .map_err(|error| rusqlite::Error::ToSqlConversionFailure(Box::new(error))) -} - #[cfg(test)] mod tests { use super::*; @@ -270,7 +228,7 @@ mod tests { "SELECT COUNT(*) FROM history_state \ WHERE singleton_id = 0 AND recovery_generation = 0 \ AND base_executed_input_count = 0 \ - AND base_safe_input_index = 0", + AND base_safe_block = 0", [], |row| row.get(0), )?; @@ -290,7 +248,7 @@ mod tests { let path = dir.path().join("atomic-init.sqlite"); let mut conn = open_writer_connection(path.to_str().expect("utf8")).expect("open"); let definitions = [baseline_migration( - Some(LifecycleCommand::Setup), + None, Some(fail_after_observing_initial_metadata), )]; let migrations = Migrations::from_slice(&definitions); @@ -314,38 +272,48 @@ mod tests { } #[test] - fn baseline_mints_distinct_uuid_v4_eras_and_initializes_known_bases() { - let setup_dir = tempfile::tempdir().expect("setup tempdir"); - let setup_path = setup_dir.path().join("sequencer.sqlite"); - let setup = Storage::initialize_for_command( - setup_path.to_str().expect("utf8"), - LifecycleCommand::Setup, - ) - .expect("initialize setup"); - let setup_history = setup.history_state().expect("setup history"); - assert_eq!(setup_history.version.recovery_generation.get(), 0); - assert_eq!(setup_history.base_executed_input_count, Some(0)); - assert_eq!(setup_history.base_safe_input_index, Some(0)); - - let rebuild_dir = tempfile::tempdir().expect("rebuild tempdir"); - let rebuild_path = rebuild_dir.path().join("sequencer.sqlite"); - let rebuild = Storage::initialize_for_command( - rebuild_path.to_str().expect("utf8"), + fn setup_registers_history_only_with_complete_baseline() { + let setup_dir = tempfile::tempdir().unwrap(); + let setup_path = setup_dir.path().join("setup.sqlite"); + let mut setup = + Storage::initialize_for_command(setup_path.to_str().unwrap(), LifecycleCommand::Setup) + .unwrap(); + assert!(matches!( + setup.history_state(), + Err(rusqlite::Error::QueryReturnedNoRows) + )); + setup + .write(|tx| { + super::super::history::initialize_history_in( + tx, + sequencer_core::history::ExecutedInputCount::ZERO, + 0, + ) + }) + .unwrap(); + let a = setup.history_state().unwrap(); + let rebuild_path = setup_dir.path().join("rebuild.sqlite"); + let mut rebuild = Storage::initialize_for_command( + rebuild_path.to_str().unwrap(), LifecycleCommand::Rebuild, ) - .expect("initialize rebuild"); - let rebuild_history = rebuild.history_state().expect("rebuild history"); - assert_eq!(rebuild_history.version.recovery_generation.get(), 0); - assert_eq!(rebuild_history.base_executed_input_count, None); - assert_eq!(rebuild_history.base_safe_input_index, None); - assert_ne!(setup_history.version.era_id, rebuild_history.version.era_id); - - let generic_dir = tempfile::tempdir().expect("generic tempdir"); - let generic_path = generic_dir.path().join("sequencer.sqlite"); - let generic = - Storage::open(generic_path.to_str().expect("utf8")).expect("initialize generic schema"); - let generic_history = generic.history_state().expect("generic history"); - assert_eq!(generic_history.base_executed_input_count, Some(0)); - assert_eq!(generic_history.base_safe_input_index, Some(0)); + .unwrap(); + assert!(matches!( + rebuild.history_state(), + Err(rusqlite::Error::QueryReturnedNoRows) + )); + rebuild + .write(|tx| { + super::super::history::initialize_history_in( + tx, + sequencer_core::history::ExecutedInputCount::new(8), + 100, + ) + }) + .unwrap(); + let b = rebuild.history_state().unwrap(); + assert_ne!(a.version.era_id, b.version.era_id); + assert_eq!(b.base_executed_input_count, 8); + assert_eq!(b.base_safe_block, 100); } } diff --git a/sequencer/src/storage/queries.rs b/sequencer/src/storage/queries.rs index 55c0be97..04eddc09 100644 --- a/sequencer/src/storage/queries.rs +++ b/sequencer/src/storage/queries.rs @@ -8,29 +8,15 @@ //! any writer role. Single-caller reads stay inline in the writer that owns //! them; only the reads reused by two or more roles live here. +#[cfg(test)] use alloy_primitives::Address; use rusqlite::{Connection, OptionalExtension, Result, Transaction, params}; use super::convert::{from_unix_ms, i64_to_u16, i64_to_u32, i64_to_u64}; use super::{BatchPolicy, WriteHead}; +#[cfg(test)] use sequencer_core::l2_tx::{DirectInput, SequencedL2Tx, ValidUserOp}; -/// Highest `offset` in the valid (non-invalidated) ordered L2-tx stream, -/// or 0 when the stream is empty. This is the global replay head — the -/// cursor a snapshot taken "now" should record, so catch-up resumes -/// strictly after it. Reads the same `valid_sequenced_l2_txs` view that -/// catch-up pages through, so the snapshot cursor and the replay query -/// always agree (and an empty batch correctly inherits the prior head -/// rather than recording genesis). -pub(super) fn valid_ordered_l2_tx_head(conn: &Connection) -> Result { - let head: Option = conn.query_row( - "SELECT MAX(offset) FROM valid_sequenced_l2_txs", - [], - |row| row.get(0), - )?; - Ok(head.map(i64_to_u64).unwrap_or(0)) -} - // ── Write-head loading ─────────────────────────────────────────────────── // // Used by ingress (initialize/resume open state) and recovery (open recovery @@ -195,6 +181,7 @@ pub(super) fn query_batch_policy(conn: &Connection) -> Result { // the row shape inside its own `query_map` closure and hands the fields to // this decoder rather than defining an intermediate struct. +#[cfg(test)] pub(super) fn decode_l2_tx_row( kind: i64, sender: Option>, diff --git a/sequencer/src/storage/recovery.rs b/sequencer/src/storage/recovery.rs index d7228b86..6b7dbd29 100644 --- a/sequencer/src/storage/recovery.rs +++ b/sequencer/src/storage/recovery.rs @@ -33,7 +33,7 @@ use super::queries::{ current_safe_block_required, current_safe_block_timestamp, last_safe_progress_ms, }; use super::safe_accepted_batches::{canonical_divergence_in, frontier_nonce}; -use super::snapshot_dumps::{batch_nonce_in, clear_pending_dumps_from_nonce_in}; +use super::snapshot_dumps::has_rollback_safe_snapshot_in; /// Outcome of a danger-zone check. /// @@ -94,7 +94,7 @@ pub enum DangerStatus { #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) struct RecoveryInspection { pub(crate) danger: DangerStatus, - pub(crate) has_finalized_snapshot: bool, + pub(crate) has_recovery_checkpoint: bool, pub(crate) has_open_tip: bool, pub(crate) current_safe_block: Option, } @@ -112,8 +112,8 @@ pub(crate) enum RecoveryMutationError { expected: DangerStatus, actual: DangerStatus, }, - #[error("cannot open the Tip without a finalized snapshot")] - MissingFinalizedSnapshot, + #[error("cannot open the Tip without a recovery checkpoint")] + MissingRecoveryCheckpoint, /// The `EnsureOpenTip` phase found a valid open Tip already present. A /// stale no-Tip decision, not a danger change; unreachable under the /// process lock, and retryable if it ever fires. @@ -259,8 +259,8 @@ impl Storage { .transaction_with_behavior(TransactionBehavior::Immediate)?; let facts = inspect_recovery_in(&tx, protocol, now_ms)?; refuse_divergence(facts.danger)?; - if !facts.has_finalized_snapshot { - return Err(RecoveryMutationError::MissingFinalizedSnapshot); + if !facts.has_recovery_checkpoint { + return Err(RecoveryMutationError::MissingRecoveryCheckpoint); } if facts.danger != DangerStatus::Safe { return Err(RecoveryMutationError::StaleDecision { @@ -297,8 +297,8 @@ impl Storage { .transaction_with_behavior(TransactionBehavior::Immediate)?; let facts = inspect_recovery_in(&tx, protocol, now_ms)?; refuse_divergence(facts.danger)?; - if !facts.has_finalized_snapshot { - return Err(RecoveryMutationError::MissingFinalizedSnapshot); + if !facts.has_recovery_checkpoint { + return Err(RecoveryMutationError::MissingRecoveryCheckpoint); } let expected = DangerStatus::TipInDanger(expected_batch_index); if facts.danger != expected { @@ -327,8 +327,8 @@ impl Storage { .transaction_with_behavior(TransactionBehavior::Immediate)?; let facts = inspect_recovery_in(&tx, protocol, now_ms)?; refuse_divergence(facts.danger)?; - if !facts.has_finalized_snapshot { - return Err(RecoveryMutationError::MissingFinalizedSnapshot); + if !facts.has_recovery_checkpoint { + return Err(RecoveryMutationError::MissingRecoveryCheckpoint); } let resynced_safe_block = facts .current_safe_block @@ -382,23 +382,15 @@ pub(super) fn inspect_recovery_in( now_ms: u64, ) -> Result { let danger = check_danger_in(conn, protocol, now_ms)?; - let has_finalized_snapshot = has_finalized_snapshot_in(conn)?; + let has_recovery_checkpoint = has_rollback_safe_snapshot_in(conn)?; Ok(RecoveryInspection { danger, - has_finalized_snapshot, + has_recovery_checkpoint, has_open_tip: has_valid_open_batch(conn)?, current_safe_block: super::queries::current_safe_block(conn)?, }) } -fn has_finalized_snapshot_in(conn: &Connection) -> Result { - conn.query_row( - "SELECT EXISTS(SELECT 1 FROM finalized_snapshot)", - [], - |row| row.get(0), - ) -} - fn check_danger_in( conn: &Connection, protocol: &ProtocolTiming, @@ -582,22 +574,9 @@ fn recover_aging_tip_inner(tx: &Transaction<'_>, danger_threshold: u64) -> Resul /// /// 1. **Cascade** from `pivot` (no-op when `None`): invalidate it and every /// successor, including the open Tip. -/// 2. **Clear doomed pending snapshots, scoped to the cascade**: delete -/// pending rows with `nonce >= pivot.nonce` — exactly the cascaded -/// batches' pendings, states the canonical replay will never reach. -/// Gold-but-unpromoted pendings (batches that landed while the process -/// was down) carry lower nonces and *survive*: catch-up resumes from a -/// fresher checkpoint, and the rows are cleaned up by the next -/// promotion's `DELETE <= max_nonce`. Scoping is load-bearing: a -/// blanket clear would arm a promote-wedge crash-loop whenever a -/// *valid in-flight* closed batch existed at clear time — its pending -/// row would be deleted while the batch stayed valid, and the lane's -/// later promotion of its landing would hit the deleted row with no -/// danger arm ever firing to heal it. With the scope, any nonce the -/// lane can later observe as accepted either has its pending row intact -/// or belongs to a post-recovery batch with a fresh row. In the -/// `RecoverTip` path the scope deletes nothing — the Tip never has a -/// pending row. Finalized is untouched (L1-confirmed bytes). +/// 2. Snapshot selection excludes invalidated batches in the same committed +/// state. Accepted batch snapshots survive; before any acceptance the +/// baseline supplies the rollback-safe restore point. /// 3. **Advance `RecoveryGeneration`** exactly once when the cascade /// invalidated any valid batch. This is the externally visible statement /// that the current era's soft-history reality changed; composing it here @@ -607,12 +586,7 @@ fn recover_aging_tip_inner(tx: &Transaction<'_>, danger_threshold: u64) -> Resul /// runtime's genesis path uses — see `ingress::open_fresh_tip_in_tx`. fn cascade_and_reopen(tx: &Transaction<'_>, pivot: Option) -> Result> { let invalidated = match pivot { - Some(batch_index) => { - let pivot_nonce = batch_nonce_in(tx, batch_index)?; - let invalidated = cascade_invalidate_from(tx, batch_index)?; - clear_pending_dumps_from_nonce_in(tx, pivot_nonce)?; - invalidated - } + Some(batch_index) => cascade_invalidate_from(tx, batch_index)?, None => Vec::new(), }; if !invalidated.is_empty() { @@ -661,7 +635,7 @@ fn first_non_gold_closed_batch(conn: &Connection) -> Result> { /// spine, so the closed frontier is at least as *old* as the Tip — whenever /// the Tip is in danger, the closed frontier is too, and cascading from the /// closed batch covers the Tip via `batch_index >= N`. (This ordering is -/// load-bearing for the pending-snapshot clear — see `docs/invariants.md`.) +/// also determines which batch snapshots remain valid.) /// /// Reads `safe_accepted_batches`, which is maintained atomically with each /// [`Storage::append_safe_inputs`] call. diff --git a/sequencer/src/storage/recovery_tests.rs b/sequencer/src/storage/recovery_tests.rs index 64ea89de..a690e026 100644 --- a/sequencer/src/storage/recovery_tests.rs +++ b/sequencer/src/storage/recovery_tests.rs @@ -11,6 +11,39 @@ use alloy_primitives::Address; use sequencer_core::l2_tx::SequencedL2Tx; use sequencer_core::protocol::ProtocolTiming; +/// Exercise the same frame-advance-before-close sequence as the live lane. +trait RecoveryFixture { + fn close_batch_at( + &mut self, + head: &mut crate::storage::WriteHead, + safe_block: u64, + ) -> rusqlite::Result<()>; +} +impl RecoveryFixture for Storage { + fn close_batch_at( + &mut self, + head: &mut crate::storage::WriteHead, + safe_block: u64, + ) -> rusqlite::Result<()> { + if head.safe_block != safe_block { + let start = self.next_undrained_safe_input_index()?; + let end: i64 = self.read(|tx| { + tx.query_row( + "SELECT COUNT(*) FROM safe_inputs WHERE block_number <= ?1", + [i64::try_from(safe_block).unwrap()], + |r| r.get(0), + ) + })?; + self.close_frame_only( + head, + safe_block, + SafeInputRange::new(start, u64::try_from(end).unwrap()), + )?; + } + self.close_frame_and_batch(head, safe_block) + } +} + mod guarded_phases { use super::*; use crate::storage::RecoveryMutationError; @@ -37,7 +70,7 @@ mod guarded_phases { fn ensure_tip_fixture( name: &str, - with_finalized_snapshot: bool, + with_recovery_checkpoint: bool, ) -> ( crate::storage::test_helpers::TestDb, Storage, @@ -57,10 +90,10 @@ mod guarded_phases { crate::storage::FrontierMode::Populate, ) .expect("seed fresh safe head"); - if with_finalized_snapshot { + if with_recovery_checkpoint { let prefix = db._dir.path().join("finalized"); storage - .insert_finalized_dump(&prefix, 0, 0) + .insert_baseline_snapshot(&prefix, ExecutedInputCount::ZERO) .expect("seed finalized snapshot"); } (db, storage, protocol) @@ -90,7 +123,7 @@ mod guarded_phases { .expect_err("missing finalized state must outrank the stale Tip decision"); assert!(matches!( error, - RecoveryMutationError::MissingFinalizedSnapshot + RecoveryMutationError::MissingRecoveryCheckpoint )); assert_eq!(open_tip_count(&storage), 1, "the existing Tip is untouched"); } @@ -175,7 +208,7 @@ mod guarded_phases { .initialize_open_state(10, SafeInputRange::empty_at(0)) .expect("initialize Tip"); storage - .close_frame_and_batch(&mut head, 10) + .close_batch_at(&mut head, 10) .expect("close cascade candidate"); storage .append_safe_inputs_with_timestamp( @@ -189,7 +222,7 @@ mod guarded_phases { .expect("advance safe head"); let prefix = db._dir.path().join("finalized"); storage - .insert_finalized_dump(&prefix, 0, 0) + .insert_baseline_snapshot(&prefix, ExecutedInputCount::ZERO) .expect("seed finalized snapshot"); (db, storage) } @@ -234,7 +267,7 @@ mod guarded_phases { let protocol = default_protocol_timing(); storage .conn - .execute("DELETE FROM finalized_snapshot", []) + .execute("DELETE FROM snapshots WHERE batch_index IS NULL", []) .expect("remove finalized snapshot fact"); let error = storage @@ -242,7 +275,7 @@ mod guarded_phases { .expect_err("missing finalized state must outrank a lagging view"); assert!(matches!( error, - RecoveryMutationError::MissingFinalizedSnapshot + RecoveryMutationError::MissingRecoveryCheckpoint )); assert_eq!(invalidated_count(&storage), 0); } @@ -267,7 +300,7 @@ mod guarded_phases { .expect("advance safe head"); let prefix = db._dir.path().join("finalized"); storage - .insert_finalized_dump(&prefix, 0, 0) + .insert_baseline_snapshot(&prefix, ExecutedInputCount::ZERO) .expect("seed finalized snapshot"); let error = storage @@ -307,7 +340,7 @@ mod guarded_phases { .expect_err("missing finalized state must refuse Tip recovery"); assert!(matches!( error, - RecoveryMutationError::MissingFinalizedSnapshot + RecoveryMutationError::MissingRecoveryCheckpoint )); assert_eq!(invalidated_count(&storage), 0); } @@ -361,7 +394,7 @@ mod invalid_batches { .close_frame_only(&mut head, 10, SafeInputRange::new(0, 1)) .expect("close frame"); storage - .close_frame_and_batch(&mut head, 10) + .close_batch_at(&mut head, 10) .expect("close batch 0"); let directs_1 = vec![StoredSafeInput { @@ -384,12 +417,14 @@ mod invalid_batches { let all = all_ordered_l2_txs(&mut storage); assert_eq!(all.len(), 2); - storage.insert_invalid_batch(0).expect("mark invalid"); + storage + .insert_invalid_batch(1) + .expect("invalidate the suffix"); let filtered = all_ordered_l2_txs(&mut storage); assert_eq!(filtered.len(), 1); match &filtered[0] { - SequencedL2Tx::Direct(d) => assert_eq!(d.payload.as_slice(), &[0xbb]), + SequencedL2Tx::Direct(d) => assert_eq!(d.payload.as_slice(), &[0xaa]), _ => panic!("expected direct input"), } } @@ -414,7 +449,7 @@ mod invalid_batches { .close_frame_only(&mut head, 10, SafeInputRange::new(0, 1)) .expect("close frame"); storage - .close_frame_and_batch(&mut head, 10) + .close_batch_at(&mut head, 10) .expect("close batch 0"); let txs = storage.ordered_l2_txs_for_batch(0).expect("load batch 0"); @@ -488,9 +523,7 @@ mod recover_post_flush { .initialize_open_state(10, SafeInputRange::empty_at(0)) .expect("initialize"); for _ in 0..3 { - storage - .close_frame_and_batch(&mut head, 10) - .expect("close batch"); + storage.close_batch_at(&mut head, 10).expect("close batch"); } let batch_submitter = Address::repeat_byte(0xAA); @@ -526,9 +559,7 @@ mod recover_post_flush { let mut head = storage .initialize_open_state(10, SafeInputRange::empty_at(0)) .expect("initialize"); - storage - .close_frame_and_batch(&mut head, 10) - .expect("close batch"); + storage.close_batch_at(&mut head, 10).expect("close batch"); let batch_submitter = Address::repeat_byte(0xAA); storage @@ -595,7 +626,7 @@ mod recover_post_flush { .initialize_open_state(10, SafeInputRange::empty_at(0)) .expect("initialize"); storage - .close_frame_and_batch(&mut head, 10) + .close_batch_at(&mut head, 10) .expect("close batch 0"); let batch_submitter = Address::repeat_byte(0xAA); @@ -628,7 +659,7 @@ mod recover_post_flush { // Submitter posts the recovery batch; it lands fresh on L1. let mut head = storage.open_state().expect("load").unwrap(); storage - .close_frame_and_batch(&mut head, 1300) + .close_batch_at(&mut head, 1300) .expect("close recovery batch"); let landed = local_batch_payload(&mut storage, 0); storage @@ -666,7 +697,7 @@ mod recover_post_flush { .initialize_open_state(10, SafeInputRange::empty_at(0)) .expect("initialize"); storage - .close_frame_and_batch(&mut head, 10) + .close_batch_at(&mut head, 10) .expect("close batch 0"); let batch_submitter = Address::repeat_byte(0xAA); @@ -687,7 +718,7 @@ mod recover_post_flush { let mut head = storage.open_state().expect("load").unwrap(); storage - .close_frame_and_batch(&mut head, 100) + .close_batch_at(&mut head, 1210) .expect("close gen2 batch"); storage @@ -695,7 +726,7 @@ mod recover_post_flush { 2410, &[StoredSafeInput { sender: batch_submitter, - payload: make_stale_batch_payload(0, 100), + payload: make_stale_batch_payload(0, 1210), block_number: 2410, }], SENDER_A, @@ -740,7 +771,7 @@ mod recover_post_flush { .initialize_open_state(10, SafeInputRange::empty_at(0)) .expect("initialize"); storage - .close_frame_and_batch(&mut head, 10) + .close_batch_at(&mut head, 10) .expect("close batch 0; open Tip with same safe_block=10"); let batch_submitter = Address::repeat_byte(0xAA); @@ -799,7 +830,7 @@ mod recover_post_flush { .initialize_open_state(10, SafeInputRange::empty_at(0)) .expect("initialize"); storage - .close_frame_and_batch(&mut head, 10) + .close_batch_at(&mut head, 10) .expect("close batch 0"); let batch_submitter = Address::repeat_byte(0xAA); @@ -982,7 +1013,7 @@ mod tip_staleness { .initialize_open_state(10, SafeInputRange::empty_at(0)) .expect("initialize at safe_block=10"); storage - .close_frame_and_batch(&mut head, 10) + .close_batch_at(&mut head, 10) .expect("close batch 0"); // Advance safe head so batch 0's first frame (safe_block=10) is stale. @@ -1007,7 +1038,7 @@ mod tip_staleness { .initialize_open_state(10, SafeInputRange::empty_at(0)) .expect("initialize"); storage - .close_frame_and_batch(&mut head, 10) + .close_batch_at(&mut head, 10) .expect("close batch 0"); storage.insert_invalid_batch(0).expect("invalidate 0"); @@ -1045,7 +1076,7 @@ mod tip_staleness { .initialize_open_state(10, SafeInputRange::empty_at(0)) .expect("initialize"); storage - .close_frame_and_batch(&mut head, 10) + .close_batch_at(&mut head, 10) .expect("close batch 0"); // Advance safe head so batch 0's first frame (safe_block=10) is stale. @@ -1113,7 +1144,7 @@ mod tip_staleness { let mut storage = Storage::open(db.path.as_str()).expect("open storage"); let mut head = storage - .initialize_open_state(10, SafeInputRange::empty_at(0)) + .initialize_open_state(0, SafeInputRange::empty_at(0)) .expect("initialize"); storage .append_safe_inputs( @@ -1136,11 +1167,10 @@ mod tip_staleness { safe_input_index: 0, executed_input_offset: ExecutedInputCount::ZERO, }], - None, ) .expect("attribute mapped direct"); storage - .close_frame_and_batch(&mut head, 10) + .close_batch_at(&mut head, 10) .expect("close batch 0"); assert_eq!( storage @@ -1194,9 +1224,7 @@ mod tip_staleness { ); let mappings: Vec = storage .conn - .prepare( - "SELECT executed_input_offset FROM executed_inputs ORDER BY executed_input_offset", - ) + .prepare("SELECT offset FROM application_inputs ORDER BY offset") .expect("prepare mapping query") .query_map([], |row| row.get(0)) .expect("query mappings") @@ -1255,7 +1283,7 @@ mod tip_staleness { .close_frame_only(&mut head, 10, SafeInputRange::new(0, 2)) .expect("close frame with deposits"); storage - .close_frame_and_batch(&mut head, 10) + .close_batch_at(&mut head, 10) .expect("close batch 0"); let before = all_ordered_l2_txs(&mut storage); @@ -1322,7 +1350,7 @@ mod tip_staleness { .initialize_open_state(10, SafeInputRange::empty_at(0)) .expect("initialize"); storage - .close_frame_and_batch(&mut head, 10) + .close_batch_at(&mut head, 10) .expect("close batch 0 with no deposits"); let non_submitter = Address::repeat_byte(0xCC); @@ -1394,7 +1422,7 @@ mod tip_staleness { .initialize_open_state(10, SafeInputRange::empty_at(0)) .expect("initialize"); storage - .close_frame_and_batch(&mut head, 10) + .close_batch_at(&mut head, 10) .expect("close batch 0"); let batch_submitter = SENDER_A; @@ -1442,7 +1470,7 @@ mod tip_staleness { .initialize_open_state(10, SafeInputRange::empty_at(0)) .expect("initialize"); storage - .close_frame_and_batch(&mut head, 10) + .close_batch_at(&mut head, 10) .expect("close batch 0 (nonce 0)"); let batch_submitter = SENDER_A; @@ -1514,7 +1542,7 @@ mod tip_staleness { .initialize_open_state(10, SafeInputRange::empty_at(0)) .expect("initialize"); storage - .close_frame_and_batch(&mut head, 10) + .close_batch_at(&mut head, 10) .expect("close batch 0"); let batch_submitter = SENDER_A; @@ -1584,7 +1612,7 @@ mod check_danger_zone { .initialize_open_state(10, SafeInputRange::empty_at(0)) .expect("initialize"); storage - .close_frame_and_batch(&mut head, 100) + .close_batch_at(&mut head, 100) .expect("close batch 0"); let landed = local_batch_payload(&mut storage, 0); @@ -1710,10 +1738,10 @@ mod check_any_unresolved { .initialize_open_state(10, SafeInputRange::empty_at(0)) .expect("initialize"); storage - .close_frame_and_batch(&mut head, 10) + .close_batch_at(&mut head, 10) .expect("close batch 0"); storage - .close_frame_and_batch(&mut head, 100) + .close_batch_at(&mut head, 100) .expect("close batch 1"); let landed = local_batch_payload(&mut storage, 0); @@ -1748,10 +1776,10 @@ mod check_any_unresolved { .initialize_open_state(10, SafeInputRange::empty_at(0)) .expect("initialize"); storage - .close_frame_and_batch(&mut head, 10) + .close_batch_at(&mut head, 10) .expect("close batch 0"); storage - .close_frame_and_batch(&mut head, 100) + .close_batch_at(&mut head, 100) .expect("close batch 1"); let landed = local_batch_payload(&mut storage, 0); @@ -1797,10 +1825,10 @@ mod check_any_unresolved { .initialize_open_state(10, SafeInputRange::empty_at(0)) .expect("initialize"); storage - .close_frame_and_batch(&mut head, 50) + .close_batch_at(&mut head, 50) .expect("seal batch 0, open batch 1 @ sb 50"); storage - .close_frame_and_batch(&mut head, 50) + .close_batch_at(&mut head, 50) .expect("seal batch 1, open the Tip @ sb 50"); // Make batch 0 gold: land its genuine bytes (accepted) → frontier → nonce 1. @@ -1854,9 +1882,7 @@ mod boundary { let mut head = storage .initialize_open_state(100, SafeInputRange::empty_at(0)) .expect("initialize"); - storage - .close_frame_and_batch(&mut head, 100) - .expect("close batch"); + storage.close_batch_at(&mut head, 100).expect("close batch"); storage .append_safe_inputs( @@ -1883,9 +1909,7 @@ mod boundary { let mut head = storage .initialize_open_state(100, SafeInputRange::empty_at(0)) .expect("initialize"); - storage - .close_frame_and_batch(&mut head, 100) - .expect("close batch"); + storage.close_batch_at(&mut head, 100).expect("close batch"); let landed = local_batch_payload(&mut storage, 0); storage @@ -1916,7 +1940,7 @@ mod boundary { .initialize_open_state(10, SafeInputRange::empty_at(0)) .expect("initialize"); for _ in 0..3 { - storage.close_frame_and_batch(&mut head, 10).expect("close"); + storage.close_batch_at(&mut head, 10).expect("close"); } storage @@ -1944,7 +1968,7 @@ mod boundary { let mut head = storage .initialize_open_state(10, SafeInputRange::empty_at(0)) .expect("initialize"); - storage.close_frame_and_batch(&mut head, 10).expect("close"); + storage.close_batch_at(&mut head, 10).expect("close"); storage .append_safe_inputs( @@ -1963,7 +1987,7 @@ mod boundary { let mut head2 = storage.open_state().expect("load").unwrap(); storage - .close_frame_and_batch(&mut head2, 1210) + .close_batch_at(&mut head2, 1210) .expect("close gen2"); storage @@ -1991,7 +2015,7 @@ mod boundary { let mut head = storage .initialize_open_state(10, SafeInputRange::empty_at(0)) .expect("init"); - storage.close_frame_and_batch(&mut head, 10).expect("close"); + storage.close_batch_at(&mut head, 10).expect("close"); storage .append_safe_inputs( 1210, @@ -2008,7 +2032,7 @@ mod boundary { let mut head2 = storage.open_state().expect("load").unwrap(); storage - .close_frame_and_batch(&mut head2, 1210) + .close_batch_at(&mut head2, 1210) .expect("close gen2"); storage .append_safe_inputs( @@ -2026,7 +2050,7 @@ mod boundary { let mut head3 = storage.open_state().expect("load").unwrap(); storage - .close_frame_and_batch(&mut head3, 2410) + .close_batch_at(&mut head3, 2410) .expect("close gen3"); let landed = local_batch_payload(&mut storage, 0); storage @@ -2054,7 +2078,7 @@ mod boundary { .initialize_open_state(10, SafeInputRange::empty_at(0)) .expect("initialize"); for _ in 0..50 { - storage.close_frame_and_batch(&mut head, 10).expect("close"); + storage.close_batch_at(&mut head, 10).expect("close"); } storage @@ -2123,7 +2147,7 @@ mod schema_invariants { .initialize_open_state(0, SafeInputRange::empty_at(0)) .expect("initialize"); storage - .close_frame_and_batch(&mut head, 0) + .close_batch_at(&mut head, 0) .expect("close batch 0; batch 1 is now Tip"); // Batch 1 has nonce 1 (0 + 1). Insert child with nonce 99 (should be 2). let err = storage.conn.execute( @@ -2275,7 +2299,7 @@ mod schema_invariants { .initialize_open_state(0, SafeInputRange::empty_at(0)) .expect("initialize"); storage - .close_frame_and_batch(&mut head, 0) + .close_batch_at(&mut head, 0) .expect("close batch 0 (seals it)"); // Batch 0 is sealed. Attempt to re-seal with a different timestamp. let err = storage.conn.execute( @@ -2318,7 +2342,7 @@ mod schema_invariants { .initialize_open_state(0, SafeInputRange::empty_at(0)) .expect("initialize"); storage - .close_frame_and_batch(&mut head, 0) + .close_batch_at(&mut head, 0) .expect("close batch 0; batch 0 is now sealed"); // Batch 0 is sealed. Any direct insert into its frames must fail. let err = storage.conn.execute( @@ -2359,9 +2383,7 @@ mod schema_invariants { let mut head = storage .initialize_open_state(0, SafeInputRange::empty_at(0)) .expect("initialize"); - storage - .close_frame_and_batch(&mut head, 0) - .expect("close batch 0"); + storage.close_batch_at(&mut head, 0).expect("close batch 0"); // Try to change parent of batch 1 — should be rejected. let err = storage.conn.execute( "UPDATE batches SET parent_batch_index = NULL WHERE batch_index = 1", @@ -2374,9 +2396,9 @@ mod schema_invariants { } #[test] - fn schema_rejects_sequenced_l2_tx_into_non_tip() { + fn schema_rejects_application_input_into_non_tip() { // The third sibling of the tip-only triggers (frames + user_ops already - // have negative tests; sequenced_l2_txs did not). The global replay order + // have negative tests; application_inputs did not). The global replay order // is the source for recovery re-drain and catch-up; a stale-WriteHead row // into a sealed batch would corrupt it. let db = temp_db("schema-sequenced-into-sealed"); @@ -2385,18 +2407,18 @@ mod schema_invariants { .initialize_open_state(0, SafeInputRange::empty_at(0)) .expect("initialize"); storage - .close_frame_and_batch(&mut head, 0) + .close_batch_at(&mut head, 0) .expect("close batch 0; it is now sealed (no longer the Tip)"); // Target the sealed batch's existing frame (0, 0). The BEFORE-INSERT // trigger fires before the safe_input_index FK is evaluated. let err = storage.conn.execute( - "INSERT INTO sequenced_l2_txs \ + "INSERT INTO application_inputs \ (offset, batch_index, frame_in_batch, user_op_pos_in_frame, safe_input_index) \ - VALUES (999, 0, 0, NULL, 0)", + VALUES (0, 0, 0, NULL, 0)", [], ); assert!( - format!("{err:?}").contains("sequenced_l2_txs can only target the current Tip"), + format!("{err:?}").contains("application input must target the current valid Tip"), "expected tip-only-sequenced trigger, got: {err:?}" ); } @@ -2409,9 +2431,7 @@ mod schema_invariants { let mut head = storage .initialize_open_state(0, SafeInputRange::empty_at(0)) .expect("initialize"); - storage - .close_frame_and_batch(&mut head, 0) - .expect("close batch 0"); + storage.close_batch_at(&mut head, 0).expect("close batch 0"); let err = storage.conn.execute( "UPDATE batches SET nonce = nonce + 1 WHERE batch_index = 0", [], @@ -2433,7 +2453,7 @@ mod schema_invariants { .initialize_open_state(0, SafeInputRange::empty_at(0)) .expect("initialize"); storage - .close_frame_and_batch(&mut head, 0) + .close_batch_at(&mut head, 0) .expect("close batch 0; payload_hash is stamped at seal"); let err = storage.conn.execute( "UPDATE batches SET payload_hash = \ @@ -2466,13 +2486,13 @@ mod schema_invariants { .initialize_open_state(10, SafeInputRange::empty_at(0)) .expect("initialize at safe_block=10"); storage - .close_frame_and_batch(&mut head, 100) + .close_batch_at(&mut head, 100) .expect("close batch 0 (nonce 0)"); storage - .close_frame_and_batch(&mut head, 100) + .close_batch_at(&mut head, 100) .expect("close batch 1 (nonce 1)"); storage - .close_frame_and_batch(&mut head, 100) + .close_batch_at(&mut head, 100) .expect("close batch 2 (nonce 2)"); // Head is now batch 3 (nonce 3, first_frame_safe_block=100). @@ -2589,8 +2609,8 @@ mod schema_invariants { } #[test] - fn schema_rejects_sequenced_l2_tx_with_neither_xor_branch() { - // `sequenced_l2_txs` must be either a user-op row + fn schema_rejects_application_input_with_neither_xor_branch() { + // `application_inputs` must be either a user-op row // (user_op_pos_in_frame IS NOT NULL) or a direct-input row // (safe_input_index IS NOT NULL), never both and never neither. // Setting both to NULL is the clean XOR violation to test — @@ -2602,14 +2622,14 @@ mod schema_invariants { .initialize_open_state(0, SafeInputRange::empty_at(0)) .expect("initialize"); let err = storage.conn.execute( - "INSERT INTO sequenced_l2_txs \ + "INSERT INTO application_inputs \ (offset, batch_index, frame_in_batch, user_op_pos_in_frame, safe_input_index) \ VALUES (0, 0, 0, NULL, NULL)", [], ); assert!( format!("{err:?}").contains("CHECK constraint failed"), - "expected CHECK constraint error on sequenced_l2_txs XOR, got: {err:?}", + "expected CHECK constraint error on application_inputs XOR, got: {err:?}", ); } @@ -2670,10 +2690,10 @@ mod schema_invariants { .initialize_open_state(10, SafeInputRange::empty_at(0)) .expect("initialize open state"); storage - .close_frame_and_batch(&mut head, 10) + .close_batch_at(&mut head, 10) .expect("close batch 0 before freezing"); storage - .insert_pending_dump(std::path::Path::new("/tmp/pending-fixture"), 1, 0) + .insert_batch_snapshot(std::path::Path::new("/tmp/pending-fixture"), 0) .expect("insert a pending dump row to attempt deleting"); crate::storage::test_helpers::record_canonical_divergence(&mut storage, 0, 0); @@ -2707,18 +2727,7 @@ mod schema_invariants { frozen( storage .conn - .execute( - "INSERT OR REPLACE INTO finalized_snapshot \ - (singleton_id, dump_id, inclusion_block, l2_tx_index) \ - VALUES (0, 999, 1, 0)", - [], - ) - .expect_err("promotion INSERT must be frozen"), - ); - frozen( - storage - .conn - .execute("DELETE FROM pending_snapshots", []) + .execute("DELETE FROM snapshots", []) .expect_err("pending-snapshot DELETE must be frozen"), ); @@ -2894,9 +2903,7 @@ mod tree_invariants { .expect("initialize"); assert_tree_invariants(&mut storage); for _ in 0..4 { - storage - .close_frame_and_batch(&mut head, 100) - .expect("close"); + storage.close_batch_at(&mut head, 100).expect("close"); assert_tree_invariants(&mut storage); } // Tree: 0(Gold sentinel in concept)→1→2→3→4 (Tip) @@ -2925,9 +2932,7 @@ mod tree_invariants { // Phase 3: more rotations after partial cascade. let mut head = storage.open_state().expect("load").unwrap(); for _ in 0..3 { - storage - .close_frame_and_batch(&mut head, 1500) - .expect("close gen2"); + storage.close_batch_at(&mut head, 1500).expect("close gen2"); assert_tree_invariants(&mut storage); } @@ -2942,9 +2947,7 @@ mod tree_invariants { // Phase 5: rotations after torn cascade — new Tip has parent=NULL, nonce=0. let mut head = storage.open_state().expect("load").unwrap(); for _ in 0..5 { - storage - .close_frame_and_batch(&mut head, 2000) - .expect("close gen3"); + storage.close_batch_at(&mut head, 2000).expect("close gen3"); assert_tree_invariants(&mut storage); } } @@ -2963,9 +2966,7 @@ mod tree_invariants { .initialize_open_state(10, SafeInputRange::empty_at(0)) .expect("initialize"); for _ in 0..4 { - storage - .close_frame_and_batch(&mut head, 100) - .expect("close"); + storage.close_batch_at(&mut head, 100).expect("close"); } let landed = local_batch_payload(&mut storage, 0); storage @@ -2987,9 +2988,7 @@ mod tree_invariants { let mut head = storage.open_state().expect("load").unwrap(); for _ in 0..2 { - storage - .close_frame_and_batch(&mut head, 1500) - .expect("close"); + storage.close_batch_at(&mut head, 1500).expect("close"); } // Assert equivalence among VALID batches for every valid N. @@ -3050,209 +3049,172 @@ mod tree_invariants { } } -mod recovery_clears_pending_snapshots { - //! Recovery's interaction with `pending_snapshots`. - //! - //! When a cascade invalidates a span of batches, their pending - //! snapshots correspond to states the canonical replay will never - //! reach. Catch-up after recovery would load one of those rows - //! and end up with state ahead of the canonical stream — a real - //! correctness issue, not hygiene. So the recovery transaction - //! clears `pending_snapshots` atomically with the cascade. - //! - //! `finalized_snapshot` is untouched: it's bytes for an - //! L1-confirmed batch, which survives any cascade. - +mod recovery_snapshot_selection { use super::*; - use std::path::PathBuf; + use std::path::Path; - fn register_finalized(storage: &mut Storage, prefix: &str) { + fn baseline(storage: &mut Storage) -> i64 { storage - .insert_finalized_dump(&PathBuf::from(format!("/tmp/test-{prefix}")), 0, 0) - .expect("insert finalized"); + .insert_baseline_snapshot( + Path::new("/tmp/recovery-baseline"), + ExecutedInputCount::ZERO, + ) + .expect("baseline") } - fn register_pending(storage: &mut Storage, nonce: u64) { + fn batch_snapshot(storage: &mut Storage, batch_index: u64) -> i64 { storage - .insert_pending_dump( - &PathBuf::from(format!("/tmp/test-pending-{nonce}")), - nonce, - 0, + .insert_batch_snapshot( + Path::new(&format!("/tmp/recovery-batch-{batch_index}")), + batch_index, ) - .expect("insert pending"); + .expect("batch snapshot") } #[test] - fn post_flush_cascade_clears_pending_dumps() { - let db = temp_db("recovery-clears-pending-post-flush"); - let mut storage = Storage::open(db.path.as_str()).expect("open storage"); - - let mut head = storage - .initialize_open_state(10, SafeInputRange::empty_at(0)) - .expect("initialize"); - for _ in 0..3 { - storage - .close_frame_and_batch(&mut head, 10) - .expect("close batch"); + fn cascade_hides_every_doomed_snapshot_and_retains_baseline() { + let db = temp_db("recovery-baseline-fallback"); + let mut storage = Storage::open(&db.path).expect("open"); + let baseline_id = baseline(&mut storage); + seed_closed_batches(&mut storage, 3); + for index in 0..3 { + batch_snapshot(&mut storage, index); } - register_finalized(&mut storage, "fin"); - register_pending(&mut storage, 0); - register_pending(&mut storage, 1); - register_pending(&mut storage, 2); - assert!(storage.latest_pending_dump().unwrap().is_some()); - - let batch_submitter = Address::repeat_byte(0xAA); + assert_ne!( + storage.latest_snapshot().unwrap().unwrap().dump.id, + baseline_id + ); storage - .append_safe_inputs( - 1210, - &[StoredSafeInput { - sender: batch_submitter, - payload: make_stale_batch_payload(0, 10), - block_number: 1210, - }], - SENDER_A, - &default_protocol_timing(), - ) - .expect("append safe input"); - - let invalidated = storage - .recover_post_flush(1200) - .expect("post-flush recover"); - assert_eq!(invalidated, vec![0, 1, 2, 3]); - - assert!( - storage.latest_pending_dump().unwrap().is_none(), - "cascade should clear pending snapshots" + .append_safe_inputs(1500, &[], SENDER_A, &default_protocol_timing()) + .expect("safe head"); + assert_eq!( + storage.recover_post_flush(1200).expect("cascade"), + vec![0, 1, 2, 3] + ); + assert_eq!( + storage.latest_snapshot().unwrap().unwrap().dump.id, + baseline_id ); assert!( - storage.finalized_dump().unwrap().is_some(), - "cascade must not touch finalized" + storage + .has_rollback_safe_snapshot() + .expect("rollback checkpoint") + ); + let garbage = storage + .gc_unreferenced_dumps() + .expect("gc doomed snapshots"); + assert_eq!(garbage.len(), 3); + assert!(garbage.iter().all(|dump| dump.id != baseline_id)); + } + + #[test] + fn accepted_snapshot_survives_cascade_after_baseline_gc_and_restart() { + let db = temp_db("recovery-accepted-fallback"); + let mut storage = Storage::open(&db.path).expect("open"); + let baseline_id = baseline(&mut storage); + seed_closed_batches(&mut storage, 2); + let accepted_id = batch_snapshot(&mut storage, 0); + let doomed_id = batch_snapshot(&mut storage, 1); + seed_safe_inputs_with_batch_nonces(&mut storage, SENDER_A, 10, &[0]); + let garbage = storage.gc_unreferenced_dumps().expect("retire baseline"); + assert_eq!( + garbage.iter().map(|d| d.id).collect::>(), + vec![baseline_id] ); - } - - #[test] - fn aging_tip_cascade_clears_pending_dumps() { - let db = temp_db("recovery-clears-pending-aging-tip"); - let mut storage = Storage::open(db.path.as_str()).expect("open storage"); - - // Initialize Tip and age it past the threshold by advancing the - // safe head well beyond the Tip's safe_block. - let _head = storage - .initialize_open_state(10, SafeInputRange::empty_at(0)) - .expect("initialize"); - register_finalized(&mut storage, "fin-aging"); - register_pending(&mut storage, 0); - assert!(storage.latest_pending_dump().unwrap().is_some()); - - let batch_submitter = Address::repeat_byte(0xAA); - storage - .append_safe_inputs(1210, &[], batch_submitter, &default_protocol_timing()) - .expect("advance safe head past threshold"); - - let invalidated = storage.recover_aging_tip(1200).expect("aging-tip recover"); - assert_eq!(invalidated, vec![0], "Tip should cascade"); - + assert_eq!( + storage.latest_snapshot().unwrap().unwrap().dump.id, + doomed_id + ); + drop(storage); + let mut storage = Storage::open(&db.path).expect("restart"); assert!( - storage.latest_pending_dump().unwrap().is_none(), - "Tip cascade should clear pending snapshots" + storage + .has_rollback_safe_snapshot() + .expect("accepted checkpoint") + ); + assert_eq!( + storage.recover_post_flush(1200).expect("cascade suffix"), + vec![1, 2] + ); + assert_eq!( + storage.latest_snapshot().unwrap().unwrap().dump.id, + accepted_id + ); + assert_eq!( + storage.finalized_dump().unwrap().unwrap().dump.id, + accepted_id + ); + assert_eq!( + storage + .gc_unreferenced_dumps() + .unwrap() + .iter() + .map(|d| d.id) + .collect::>(), + vec![doomed_id] ); assert!( - storage.finalized_dump().unwrap().is_some(), - "Tip cascade must not touch finalized" + storage + .recover_post_flush(1200) + .expect("idempotent recovery") + .is_empty() ); } #[test] - fn no_op_recovery_does_not_touch_pending_dumps() { - // When recovery has nothing to invalidate (closed batches are - // all "gold" — their nonces are below the scheduler's - // frontier — and the Tip isn't aged), pending snapshots must - // stay: their batches are in-flight, the snapshots still - // useful for catch-up. - let db = temp_db("recovery-noop-keeps-pending"); - let mut storage = Storage::open(db.path.as_str()).expect("open storage"); - - let mut head = storage - .initialize_open_state(10, SafeInputRange::empty_at(0)) - .expect("initialize"); + fn an_optimistic_snapshot_alone_does_not_admit_recovery() { + let db = temp_db("recovery-optimistic-is-not-fallback"); + let mut storage = Storage::open(&db.path).expect("open"); + seed_closed_batches(&mut storage, 1); + batch_snapshot(&mut storage, 0); + let protocol = default_protocol_timing(); + let now = crate::clock::unix_now_ms(); storage - .close_frame_and_batch(&mut head, 10) - .expect("close batch"); - - // Acknowledge our batch (nonce 0) at L1, advancing the gold - // frontier to nonce 1 so the closed batch we just made falls - // "below" frontier and isn't a cascade pivot. - let batch_submitter = Address::repeat_byte(0xAA); - seed_safe_inputs_with_batch_nonces(&mut storage, batch_submitter, 10, &[0]); - - register_finalized(&mut storage, "fin-noop"); - register_pending(&mut storage, 1); - - let post_flush_invalidated = storage.recover_post_flush(1200).expect("post-flush no-op"); - let aging_tip_invalidated = storage.recover_aging_tip(1200).expect("aging-tip no-op"); - - assert!( - post_flush_invalidated.is_empty(), - "post-flush should be a no-op" - ); + .append_safe_inputs_with_timestamp( + 1500, + now / 1000, + &[], + SENDER_A, + &protocol, + crate::storage::FrontierMode::Populate, + ) + .expect("fresh observation"); assert!( - aging_tip_invalidated.is_empty(), - "aging-tip should be a no-op" + storage + .latest_snapshot() + .expect("optimistic snapshot") + .is_some() ); assert!( - storage.latest_pending_dump().unwrap().is_some(), - "no-op recovery must preserve in-flight pending snapshots" + !storage + .has_rollback_safe_snapshot() + .expect("no stable fallback") ); - } - - /// Regression test: the cascade's pending clear is scoped to - /// `nonce >= pivot.nonce`. A gold-but-unpromoted pending (its batch - /// landed accepted while the process was down, the lane never - /// promoted it) sits *below* the pivot and must survive — deleting - /// it would arm a promote-wedge crash-loop when the lane later - /// observes the landing and promotion hits the deleted row. - #[test] - fn cascade_preserves_gold_unpromoted_pending_below_pivot() { - let db = temp_db("recovery-scoped-clear-preserves-gold"); - let mut storage = Storage::open(db.path.as_str()).expect("open storage"); - - let mut head = storage - .initialize_open_state(10, SafeInputRange::empty_at(0)) - .expect("initialize"); - // Close two batches: nonce 0 (will land gold) and nonce 1 (doomed). - storage - .close_frame_and_batch(&mut head, 10) - .expect("close batch nonce 0"); - storage - .close_frame_and_batch(&mut head, 10) - .expect("close batch nonce 1"); - - register_finalized(&mut storage, "fin-scoped"); - register_pending(&mut storage, 0); // gold-but-unpromoted - register_pending(&mut storage, 1); // doomed (past the frontier) - - // Batch nonce 0 lands accepted: gold frontier advances to 1, so - // the post-flush cascade pivots at the nonce-1 batch. - let batch_submitter = Address::repeat_byte(0xAA); - seed_safe_inputs_with_batch_nonces(&mut storage, batch_submitter, 10, &[0]); - - let invalidated = storage - .recover_post_flush(1200) - .expect("post-flush recover"); + let err = storage + .recover_post_flush_for_recovery(1500, &protocol, now) + .expect_err("refuse before losing only snapshot"); + assert!(matches!( + err, + crate::storage::RecoveryMutationError::MissingRecoveryCheckpoint + )); assert_eq!( - invalidated, - vec![1, 2], - "cascade covers the nonce-1 batch and the Tip" + storage + .latest_snapshot() + .unwrap() + .unwrap() + .executed_input_count, + ExecutedInputCount::ZERO ); - - let survivor = storage - .latest_pending_dump() - .unwrap() - .expect("gold-unpromoted pending must survive the scoped clear"); - assert_eq!(survivor.nonce, 0, "the survivor is the gold pending"); - assert!( - storage.finalized_dump().unwrap().is_some(), - "cascade must not touch finalized" + assert_eq!( + storage + .conn + .query_row( + "SELECT COUNT(*) FROM batches WHERE invalidated_at_ms IS NOT NULL", + [], + |r| r.get::<_, i64>(0) + ) + .unwrap(), + 0 ); } } diff --git a/sequencer/src/storage/safe_accepted_batches.rs b/sequencer/src/storage/safe_accepted_batches.rs index 710d39f1..7d37fffd 100644 --- a/sequencer/src/storage/safe_accepted_batches.rs +++ b/sequencer/src/storage/safe_accepted_batches.rs @@ -59,7 +59,8 @@ pub(super) fn query_latest_safe_accepted_batch( /// Next nonce the scheduler is expected to accept — the gold frontier's /// "expected next" cursor. /// -/// Returns `latest_accepted.nonce + 1` if any batch has been accepted, else `0`. +/// Returns `latest_accepted.nonce + 1` if any batch has been accepted, else the +/// deployment anchor (`0` for genesis, `N'` after rebuild). /// Equivalently, the nonce that the very next valid closed batch (the cascade /// pivot, when one exists) will carry, by the contiguity invariant on the /// valid path (`trg_enforce_nonce_contiguity`). @@ -91,7 +92,7 @@ fn next_expected_nonce(nonce: u64) -> u64 { /// matches to `safe_accepted_batches`. /// /// The content-identity check (I9/I15) is complete for this mirrored -/// predicate: every at/above-anchor accepted landing is a byte-identical +/// predicate: every post-baseline accepted landing is a byte-identical /// local match, foreign, or mismatched. It is not /// an independent oracle for the canonical scheduler, application state, or /// collapsed checkpoint history; foreign/mismatch requires manual cockroach @@ -136,14 +137,15 @@ pub(super) fn populate_safe_accepted_batches( const SELECT_SQL: &str = "SELECT safe_input_index, payload, block_number \ FROM safe_inputs \ WHERE sender = ?1 AND safe_input_index > ?2 \ - ORDER BY safe_input_index ASC LIMIT ?3"; + AND block_number > ?3 \ + ORDER BY safe_input_index ASC LIMIT ?4"; const INSERT_SQL: &str = "INSERT INTO safe_accepted_batches \ (safe_input_index, nonce, first_frame_safe_block, inclusion_block) \ VALUES (?1, ?2, ?3, ?4)"; // A persisted divergence marker freezes the acceptance frontier: the // local batch tree is no longer a reliable mirror of canonical state, - // and advancing it (or promoting on it) would compound the divergence. + // and advancing it would compound the divergence. // `check_danger` reports `CanonicalDivergence` ahead of every other arm, // so the detector exits / startup refuses; the remedy is cockroach // recovery, never standard recovery. @@ -151,16 +153,29 @@ pub(super) fn populate_safe_accepted_batches( return Ok(()); } - // The frontier begins at the batch-tree anchor: 0 for a genesis - // deployment (unchanged), or N' for a cockroach-recovered one. Below the - // anchor the local tree has no batches — that history is folded into the - // recovered checkpoint `S'`, not kept as tree batches — so those L1 - // landings are *trusted collapsed history*, not foreign. Seeding `expected` - // at the anchor makes the scan skip them by nonce-mismatch (they never - // reach the content-identity check), while landings at/above the anchor — - // the resumed instance's own batches — are accepted and checked normally. - // (See I16 / docs/recovery: the anchor is where the local tree's authority - // begins.) + // The recovered prefix is opaque. Replaying it with the final nonce N' + // could accept an old future-nonce submission that the scheduler rejected + // at its original position. Only inputs after the baseline belong here. + let base_safe_block: Option = conn + .query_row( + "SELECT base_safe_block FROM history_state WHERE singleton_id = 0", + [], + |row| row.get(0), + ) + .optional()?; + let base_safe_block = match base_safe_block { + Some(block) => block, + None => { + let setup_complete: bool = + conn.query_row("SELECT EXISTS(SELECT 1 FROM setup_complete)", [], |row| { + row.get(0) + })?; + assert!(!setup_complete, "completed setup has no history baseline"); + // Plain setup detects prior activity before installing genesis. + // Rebuild ingestion defers this projection until its baseline exists. + 0 + } + }; let anchor = batch_tree_anchor_in(conn)?; let latest_accepted = query_latest_safe_accepted_batch(conn)?; let mut cursor = latest_accepted @@ -178,7 +193,12 @@ pub(super) fn populate_safe_accepted_batches( let page: Vec<(i64, Vec, i64)> = { let mut stmt = conn.prepare_cached(SELECT_SQL)?; stmt.query_map( - params![batch_submitter.as_slice(), cursor, PAGE_SIZE,], + params![ + batch_submitter.as_slice(), + cursor, + base_safe_block, + PAGE_SIZE + ], |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), )? .collect::>()? @@ -358,6 +378,126 @@ mod tests { use super::*; use crate::storage::{Storage, test_helpers::temp_db}; + #[test] + fn recovered_prefix_does_not_reinterpret_a_rejected_future_nonce() { + use crate::storage::test_helpers::{ + default_protocol_timing, local_batch_payload, pin_test_deployment_identity, + }; + use crate::storage::{FrontierMode, LifecycleCommand, StoredSafeInput}; + use sequencer_core::{batch::Batch, history::ExecutedInputCount}; + use ssz::Encode; + + let db = temp_db("accepted-recovered-prefix"); + let mut storage = Storage::initialize_for_command(&db.path, LifecycleCommand::Rebuild) + .expect("initialize rebuild"); + let submitter = Address::repeat_byte(0x99); + let timing = default_protocol_timing(); + pin_test_deployment_identity(&mut storage, submitter); + let old_future = Batch { + nonce: 1, + frames: vec![], + } + .as_ssz_bytes(); + let old_accepted = Batch { + nonce: 0, + frames: vec![], + } + .as_ssz_bytes(); + assert!( + timing + .scheduler_accepts( + submitter, + SafeInputView { + safe_input_index: 0, + sender: submitter, + payload: &old_future, + inclusion_block: 20, + }, + 0 + ) + .is_none() + ); + assert!( + timing + .scheduler_accepts( + submitter, + SafeInputView { + safe_input_index: 1, + sender: submitter, + payload: &old_accepted, + inclusion_block: 30, + }, + 0 + ) + .is_some() + ); + storage + .append_safe_inputs_with_timestamp( + 30, + 30, + &[ + StoredSafeInput { + sender: submitter, + payload: old_future, + block_number: 20, + }, + StoredSafeInput { + sender: submitter, + payload: old_accepted, + block_number: 30, + }, + ], + submitter, + &timing, + FrontierMode::DeferUntilAnchorSet, + ) + .expect("ingest opaque prefix"); + storage + .write(|tx| { + super::super::history::initialize_history_in(tx, ExecutedInputCount::new(41), 30)?; + super::super::mutations::set_batch_tree_anchor_in(tx, 1)?; + super::super::ingress::open_recovery_tip_in_tx(tx, 30) + }) + .expect("install recovered baseline"); + let mut head = storage.open_state().expect("read root").expect("root"); + storage + .close_frame_and_batch(&mut head, 30) + .expect("close resumed batch"); + let payload = local_batch_payload(&mut storage, 1); + storage + .append_safe_inputs( + 31, + &[StoredSafeInput { + sender: submitter, + payload, + block_number: 31, + }], + submitter, + &timing, + ) + .expect("accept post-baseline batch"); + assert!( + storage + .canonical_divergence() + .expect("divergence") + .is_none() + ); + let accepted = query_latest_safe_accepted_batch(&storage.conn) + .expect("accepted frontier") + .expect("resumed acceptance"); + assert_eq!((accepted.safe_input_index, accepted.nonce), (2, 1)); + assert_eq!( + storage + .conn + .query_row("SELECT COUNT(*) FROM safe_accepted_batches", [], |row| row + .get::<_, i64>( + 0 + )) + .expect("accepted count"), + 1 + ); + } + fn insert_safe_input_zero(storage: &Storage) { storage .conn diff --git a/sequencer/src/storage/snapshot_dumps.rs b/sequencer/src/storage/snapshot_dumps.rs index 541a2468..de858c9e 100644 --- a/sequencer/src/storage/snapshot_dumps.rs +++ b/sequencer/src/storage/snapshot_dumps.rs @@ -1,61 +1,40 @@ // (c) Cartesi and individual authors (see AUTHORS) // SPDX-License-Identifier: Apache-2.0 (see LICENSE) -//! Storage half of the snapshot dump lifecycle: pending and finalized -//! snapshots plus lease management. See `migrations/0001_schema.sql` for -//! the schema rationale. -//! -//! This module exposes SQLite operations only; filesystem cleanup of the -//! actual dump artifacts is the caller's responsibility: -//! `gc_unreferenced_dumps` deletes unreferenced rows in one transaction -//! and returns their prefixes, and the lane (`inclusion_lane/snapshot.rs`) -//! removes the directories afterward. The lane drives the lifecycle — -//! register at batch close, promote on L1 observation (atomically with the -//! drain), GC after each promotion. +//! Immutable batch-close and baseline snapshots, derived acceptance, and leases. +//! Files are durable before registration; GC deletes rows before their files. use std::path::{Path, PathBuf}; use std::sync::Arc; -use rusqlite::{OptionalExtension, Result, Transaction, params}; +use rusqlite::{Connection, OptionalExtension, Result, Transaction, params}; +use sequencer_core::history::{ExecutedInputCount, HistoryVersion}; use super::convert::{i64_to_u64, u64_to_i64}; -use super::history::{bind_history_base_in, next_executed_input_count_in, query_history_state}; +use super::history::{next_executed_input_count_in, query_history_state}; use super::{Storage, is_persistent_storage_error, is_persistent_storage_open_error}; -use sequencer_core::history::{ExecutedInputCount, HistoryVersion}; -/// A row in `dumps`: SQLite primary key plus the on-disk directory. #[derive(Debug, Clone, PartialEq, Eq)] pub struct DumpRow { pub id: i64, pub prefix: PathBuf, } -/// One row of `pending_snapshots` joined with the underlying `dumps` row. #[derive(Debug, Clone, PartialEq, Eq)] -pub struct PendingDump { - pub nonce: u64, +pub struct Snapshot { pub dump: DumpRow, - pub l2_tx_index: u64, pub executed_input_count: ExecutedInputCount, } -/// The singleton `finalized_snapshot` row joined with the underlying -/// `dumps` row. +/// A batch-close snapshot whose complete predicted prefix was accepted on L1. #[derive(Debug, Clone, PartialEq, Eq)] pub struct FinalizedDump { pub dump: DumpRow, pub inclusion_block: u64, - pub l2_tx_index: u64, + pub next_batch_nonce: u64, pub executed_input_count: ExecutedInputCount, } -/// How a [`LeaseGuard`] schedules its blocking release on drop. Injected by the -/// caller so storage stays runtime-agnostic: the egress HTTP layer passes a -/// supervised queue, while sync callers and tests pass one that runs inline. -/// The owned callback lets each guard retain the queue's producer token until -/// its `Drop` has submitted the release. A separate required reporter carries -/// only the persistent-failure signal back to the runtime, keeping storage -/// independent of runtime shutdown types. pub type ReleaseScheduler = Arc) + Send + Sync + 'static>; pub type PersistentReleaseFailureReporter = Arc; @@ -109,657 +88,355 @@ impl Drop for LeaseGuard { } } -/// A leased dump: the data the egress handler needs, plus an armed release. -/// Returned by [`Storage::acquire_finalized_lease`] (inside a -/// [`FinalizedLease`]) and [`Storage::acquire_latest_snapshot_lease`] — you -/// cannot obtain the data without the guard, so there is no code path with a -/// held lease and no armed release. +/// Artifact and history metadata selected atomically with the lease increment. pub struct LeasedDump { pub prefix: PathBuf, - pub l2_tx_index: u64, pub executed_input_count: ExecutedInputCount, - /// History in which this artifact was selected, captured with its lease. pub history_version: HistoryVersion, + pub next_batch_nonce: u64, pub guard: LeaseGuard, } -/// The finalized snapshot's lease: the leased dump plus the inclusion block -/// its row carries. The column is `NOT NULL` at the engine, so the value is -/// never optional here; the latest-snapshot lease carries no block. pub struct FinalizedLease { pub inclusion_block: u64, pub dump: LeasedDump, } impl Storage { - /// Insert a new dump row plus its pending-snapshot row in one - /// transaction. Caller has already created the dump on disk at - /// `prefix`. Returns the newly assigned `dump_id`. - /// - /// Fails with a UNIQUE constraint violation if `prefix` already - /// exists in `dumps`; this is intentional — the caller is expected - /// to pass fresh, unique prefixes per call, and reuse is a bug - /// worth surfacing loudly. - /// Test seed only: production stages pending rows exclusively through - /// the lane's atomic `close_batch_with_snapshot` path. - #[cfg(test)] - pub(crate) fn insert_pending_dump( - &mut self, - prefix: &Path, - nonce: u64, - l2_tx_index: u64, - ) -> Result { - self.write(|tx| { - let executed_input_count = next_executed_input_count_in(tx)?; - insert_pending_dump_in(tx, prefix, nonce, l2_tx_index, executed_input_count) - }) - } - - /// Atomically promote the pending dump for `max_nonce` into the - /// single `finalized_snapshot` row, carrying over its - /// `l2_tx_index`. In the same transaction, every pending row with - /// `nonce <= max_nonce` is deleted — that's the explicitly-promoted - /// row plus any stale rows behind it from earlier promotions that - /// failed to clean up. L1 wallet nonces guarantee batches land in - /// monotonic order, so anything with a smaller nonce has landed - /// by the time we're observing `max_nonce`. - /// - /// `max_nonce` must currently exist in `pending_snapshots`; a - /// missing row surfaces as `QueryReturnedNoRows`. - /// - /// Standalone promotion, retained for test setup (it's the only way to - /// *supersede* an existing finalized row, which `insert_finalized_dump` - /// can't). **Production does not call this**: the lane promotes via - /// `promote_finalized_in` folded into the safe-frontier-advance - /// transaction - /// ([`Storage::close_frame_only_with_executions`]), so promotion, - /// drain, and canonical attribution commit atomically. A separate - /// promotion could commit ahead of the drain and wedge a restart on a - /// deleted pending row. - #[cfg(test)] - pub(crate) fn promote_finalized(&mut self, max_nonce: u64, inclusion_block: u64) -> Result<()> { - self.write(|tx| promote_finalized_in(tx, max_nonce, inclusion_block)) - } - - /// Increment a dump's lease count, non-atomically with any row read. - /// Test-only: production leases atomically with the row read via - /// [`Storage::acquire_finalized_lease`] / - /// [`Storage::acquire_latest_snapshot_lease`], which call - /// `acquire_dump_lease_in` inside the read transaction. - #[cfg(test)] - pub fn acquire_dump_lease(&mut self, dump_id: i64) -> Result<()> { - self.write(|tx| acquire_dump_lease_in(tx, dump_id)) - } - - /// Decrement a dump's lease count. Called by [`LeaseGuard`]'s `Drop` (via - /// the injected scheduler) when a leased stream completes, errors, or - /// disconnects. pub fn release_dump_lease(&mut self, dump_id: i64) -> Result<()> { self.write(|tx| release_dump_lease_in(tx, dump_id)) } - /// Reset every dump's lease count to zero. Called at process - /// startup to clear stale leases from a crashed previous run. pub fn reset_dump_leases(&mut self) -> Result { - self.write(reset_dump_leases_in) + self.write(|tx| tx.execute("UPDATE dumps SET lease_count = 0", [])) } - /// Current lease count for a dump, or `None` if the row doesn't exist. - /// Read-only; exposed for operational visibility (in-flight stream - /// count) and tests. pub fn dump_lease_count(&mut self, dump_id: i64) -> Result> { self.read(|tx| { tx.query_row( "SELECT lease_count FROM dumps WHERE id = ?1", - params![dump_id], - |row| row.get(0), + [dump_id], + |r| r.get(0), ) .optional() }) } - /// Return dumps eligible for garbage collection (`lease_count = 0` - /// AND not referenced by `pending_snapshots` or `finalized_snapshot`) - /// without deleting them. Test-only: production reads and deletes in - /// one transaction via [`Storage::gc_unreferenced_dumps`], closing the - /// race a read-then-delete split would open against a concurrent - /// `acquire_dump_lease_in`. - #[cfg(test)] - pub fn gc_dump_rows(&mut self) -> Result> { - self.read(gc_dump_rows_in) - } - - /// Atomic GC pass: in one SQLite transaction, find all eligible - /// dumps and delete their rows. Returns the (id, prefix) pairs so - /// the caller can drive filesystem cleanup separately — file - /// deletion is best-effort, and an orphan file on failure is - /// acceptable per the no-dangling-row invariant. - /// - /// The single-transaction shape closes a race window: between a - /// non-atomic `gc_dump_rows()` and the per-row `delete_dump_row`, - /// an HTTP handler on another thread could `acquire_dump_lease`, - /// and a naive per-row delete would race against the lease. - /// Doing both inside one `write` (Immediate-mode tx) serializes - /// against any concurrent writer. + /// GC keeps the accepted rollback checkpoint and every valid snapshot beyond + /// it. Intermediate optimistic snapshots may become the next accepted head. + /// Row selection/deletion serializes with lease acquisition; files follow. pub fn gc_unreferenced_dumps(&mut self) -> Result> { self.write(|tx| { - let candidates = gc_dump_rows_in(tx)?; - for row in &candidates { - tx.execute("DELETE FROM dumps WHERE id = ?1", params![row.id])?; + let anchor = rollback_snapshot_in(tx)?; + let anchor_id = anchor.as_ref().map(|s| s.dump.id); + let accepted_nonce = + latest_accepted_boundary_in(tx)?.map(|(_, nonce, _)| u64_to_i64(nonce)); + let candidates = { + let mut stmt = tx.prepare( + "SELECT d.id, d.prefix FROM dumps d \ + LEFT JOIN snapshots s ON s.dump_id = d.id \ + LEFT JOIN valid_closed_batches b ON b.batch_index = s.batch_index \ + WHERE d.lease_count = 0 AND (?1 IS NULL OR d.id != ?1) \ + AND (b.batch_index IS NULL OR (?2 IS NOT NULL AND b.nonce <= ?2)) \ + ORDER BY d.id", + )?; + stmt.query_map(params![anchor_id, accepted_nonce], row_to_dump_row)? + .collect::>>()? + }; + for dump in &candidates { + tx.execute("DELETE FROM snapshots WHERE dump_id = ?1", [dump.id])?; + tx.execute("DELETE FROM dumps WHERE id = ?1", [dump.id])?; } Ok(candidates) }) } - /// Delete a dump row from `dumps` by id. Errors if the row is still - /// FK-referenced. Test-only: production deletes unreferenced rows - /// atomically via [`Storage::gc_unreferenced_dumps`]. - #[cfg(test)] - pub fn delete_dump_row(&mut self, dump_id: i64) -> Result<()> { - self.write(|tx| delete_dump_row_in(tx, dump_id)) + /// The latest accepted batch must have its own snapshot. A missing artifact + /// is corruption, never a request to use an older accepted snapshot. + pub fn finalized_dump(&mut self) -> Result> { + self.read(|tx| finalized_dump_in(tx)) } - /// Read the pending snapshot with the highest nonce, if any. Test-only: - /// production reads through [`Storage::latest_snapshot`] (pending else - /// finalized); this bare pending read is only used by tests. - #[cfg(test)] - pub fn latest_pending_dump(&mut self) -> Result> { - self.read(latest_pending_dump_in) + pub fn latest_snapshot(&mut self) -> Result> { + self.read(|tx| latest_snapshot_in(tx)) } - /// Read the singleton finalized snapshot, if any. Used by the - /// `/finalized_state/inclusion_block` endpoint and as `latest_snapshot`'s - /// fallback when no pending snapshot exists. - pub fn finalized_dump(&mut self) -> Result> { - self.read(finalized_dump_in) + pub fn rollback_snapshot(&mut self) -> Result> { + self.read(|tx| rollback_snapshot_in(tx)) } - /// The snapshot to resume or serve from: the latest pending dump, else the - /// finalized snapshot. Returns its `(dump row, l2_tx_index, - /// executed_input_count)`, or `None` if neither exists. This is catch-up's resume checkpoint; the leasing variant - /// [`Storage::acquire_latest_snapshot_lease`] shares the same "pending else - /// finalized" selection via `latest_snapshot_in`. - pub fn latest_snapshot(&mut self) -> Result> { - self.read(latest_snapshot_in) + pub fn has_rollback_safe_snapshot(&mut self) -> Result { + self.read(|tx| has_rollback_safe_snapshot_in(tx)) } - /// Atomically read the finalized snapshot and history version, and lease - /// its dump, bundled with an armed release ([`LeaseGuard`]). Closes the race where a - /// handler reads the row, a promotion + GC delete the dump, and the open - /// then fails: the lease is held from the moment of the read. `None` if no - /// finalized snapshot exists. `schedule` controls where the (blocking) - /// release runs on drop — see [`ReleaseScheduler`]. The reporter is - /// required — an unreported persistent release failure must be impossible; - /// it is called only for persistent row/schema failures, never - /// BUSY/I/O. Tests pass a no-op closure. pub fn acquire_finalized_lease( &mut self, schedule: ReleaseScheduler, report_persistent_failure: PersistentReleaseFailureReporter, ) -> Result> { - let path = self.path.clone(); let acquired = self.write(|tx| { - let Some(finalized) = finalized_dump_in(tx)? else { + let Some(snapshot) = finalized_dump_in(tx)? else { return Ok(None); }; let history_version = query_history_state(tx)?.version; - acquire_dump_lease_in(tx, finalized.dump.id)?; - Ok(Some((finalized, history_version))) + acquire_dump_lease_in(tx, snapshot.dump.id)?; + Ok(Some((snapshot, history_version))) })?; - - Ok(acquired.map( - |( - FinalizedDump { - dump, - inclusion_block, - l2_tx_index, - executed_input_count, - }, + Ok(acquired.map(|(snapshot, history_version)| FinalizedLease { + inclusion_block: snapshot.inclusion_block, + dump: LeasedDump { + prefix: snapshot.dump.prefix, + executed_input_count: snapshot.executed_input_count, + next_batch_nonce: snapshot.next_batch_nonce, history_version, - )| FinalizedLease { - inclusion_block, - dump: LeasedDump { - prefix: dump.prefix, - l2_tx_index, - executed_input_count, - history_version, - // Arm the release only after `Storage::write` has committed - // the increment. A failed COMMIT rolls back the lease and - // must not schedule a decrement for a lease that never - // existed. - guard: LeaseGuard { - path, - dump_id: dump.id, - schedule, - report_persistent_failure, - }, + // Only a committed increment arms a release. A failed COMMIT + // rolls back the lease and must never schedule its decrement. + guard: LeaseGuard { + path: self.path.clone(), + dump_id: snapshot.dump.id, + schedule, + report_persistent_failure, }, }, - )) + })) } - /// Atomically read the snapshot to serve (latest pending, else finalized) - /// AND lease its dump, returning it bundled with an armed release. Same - /// contract as [`Storage::acquire_finalized_lease`], without the - /// inclusion block (the `/latest_snapshot` consumer has no use for it). pub fn acquire_latest_snapshot_lease( &mut self, schedule: ReleaseScheduler, report_persistent_failure: PersistentReleaseFailureReporter, ) -> Result> { - let path = self.path.clone(); let acquired = self.write(|tx| { - let Some((dump, l2_tx_index, executed_input_count)) = latest_snapshot_in(tx)? else { + let Some(snapshot) = latest_snapshot_in(tx)? else { return Ok(None); }; let history_version = query_history_state(tx)?.version; - let dump_id = dump.id; - acquire_dump_lease_in(tx, dump_id)?; - Ok(Some(( - dump, - l2_tx_index, - executed_input_count, - history_version, - ))) + let next_batch_nonce = snapshot_next_nonce_in(tx, snapshot.dump.id)?; + acquire_dump_lease_in(tx, snapshot.dump.id)?; + Ok(Some((snapshot, history_version, next_batch_nonce))) })?; - - Ok(acquired.map( - |(dump, l2_tx_index, executed_input_count, history_version)| LeasedDump { - prefix: dump.prefix, - l2_tx_index, - executed_input_count, + Ok( + acquired.map(|(snapshot, history_version, next_batch_nonce)| LeasedDump { + prefix: snapshot.dump.prefix, + executed_input_count: snapshot.executed_input_count, history_version, - // See `acquire_finalized_lease`: the guard owns a release only - // after the matching increment is durable. + next_batch_nonce, guard: LeaseGuard { - path, - dump_id: dump.id, + path: self.path.clone(), + dump_id: snapshot.dump.id, schedule, report_persistent_failure, }, - }, - )) + }), + ) } - /// Return every row in `dumps`. Used at startup to reconcile - /// SQLite-tracked prefixes against what actually exists on disk - /// (paths on disk not in `dumps` are removed; rows in `dumps` - /// whose paths are missing are dropped on the next GC pass). pub fn list_dump_rows(&mut self) -> Result> { - self.read(list_dump_rows_in) + self.read(|tx| { + let mut stmt = tx.prepare("SELECT id, prefix FROM dumps ORDER BY id")?; + stmt.query_map([], row_to_dump_row)?.collect() + }) } - /// Delete every row from `pending_snapshots`. Test-only convenience wrapper - /// for the *unscoped* clear: production danger-zone recovery instead composes - /// the pivot-scoped `clear_pending_dumps_from_nonce_in` into the same - /// transaction as the cascade invalidation (see `storage/recovery.rs`), so - /// only the cascade-doomed batches' pending rows are cleared, atomically with - /// them. - #[cfg(test)] - pub fn clear_pending_dumps(&mut self) -> Result { - self.write(clear_pending_dumps_in) + pub fn batch_nonce(&mut self, batch_index: u64) -> Result { + self.read(|tx| batch_nonce_in(tx, batch_index)) } - /// Insert a new dump row and a finalized-snapshot row in one - /// transaction. Used at first startup to register the genesis dump - /// directly as finalized (bypassing pending). Fails if a finalized - /// row already exists (the singleton constraint). - /// Test seed only: production registers the genesis/recovery snapshot - /// through `insert_initial_finalized_dump`, which binds the canonical - /// coordinates atomically (this branch replaced both former - /// production callers). #[cfg(test)] - pub(crate) fn insert_finalized_dump( - &mut self, - prefix: &Path, - inclusion_block: u64, - l2_tx_index: u64, - ) -> Result { + pub(crate) fn insert_batch_snapshot(&mut self, prefix: &Path, batch_index: u64) -> Result { self.write(|tx| { - let executed_input_count = next_executed_input_count_in(tx)?; - insert_finalized_dump_in( - tx, - prefix, - inclusion_block, - l2_tx_index, - executed_input_count, - ) + insert_batch_snapshot_in(tx, prefix, batch_index, next_executed_input_count_in(tx)?) }) } - /// Establish the era's application-history base, durable safe-input drain - /// floor, and initial finalized snapshot in one transaction. Plain setup - /// reasserts the migration's zero bases; cockroach setup binds the folded - /// application's absolute executed-input count and the recovery root's - /// exclusive safe-input cursor for the first and only time. - pub(crate) fn insert_initial_finalized_dump( + #[cfg(test)] + pub(crate) fn insert_baseline_snapshot( &mut self, prefix: &Path, - inclusion_block: u64, - l2_tx_index: u64, - base_executed_input_count: u64, - base_safe_input_index: u64, + count: ExecutedInputCount, ) -> Result { - self.write(|tx| { - bind_history_base_in(tx, base_executed_input_count, base_safe_input_index)?; - insert_finalized_dump_in( - tx, - prefix, - inclusion_block, - l2_tx_index, - ExecutedInputCount::new(base_executed_input_count), - ) - }) - } - - /// Highest `offset` in the valid ordered L2-tx stream (the global - /// replay head), or 0 when empty. The lane reads this before writing - /// a dump's `info.toml` at batch close; the seal transaction - /// re-asserts the same value (see - /// `close_frame_and_batch_with_pending_dump`), which is sound because - /// the lane is the single writer and nothing sequences in between. - pub fn valid_ordered_l2_tx_head(&mut self) -> Result { - self.read(|tx| super::queries::valid_ordered_l2_tx_head(tx)) - } - - /// Look up a batch's nonce from `batches`. Errors with - /// `QueryReturnedNoRows` if the batch doesn't exist — the lane - /// calls this immediately after `close_frame_and_batch` so the row - /// should always be there. - pub fn batch_nonce(&mut self, batch_index: u64) -> Result { - self.read(|tx| batch_nonce_in(tx, batch_index)) - } - - /// Look up the nonce of a previously-accepted batch by its safe - /// input index. Returns `None` if the safe input is either a - /// third-party direct input (not our batch) or one of our batches - /// that the scheduler did not accept (stale-nonce drop). Used when - /// processing safe inputs to decide whether to promote a pending - /// snapshot. - pub fn accepted_batch_nonce_at(&mut self, safe_input_index: u64) -> Result> { - self.read(|tx| { - tx.query_row( - "SELECT nonce FROM safe_accepted_batches WHERE safe_input_index = ?1", - params![u64_to_i64(safe_input_index)], - |row| row.get::<_, i64>(0), - ) - .optional() - .map(|opt| opt.map(i64_to_u64)) - }) + self.write(|tx| insert_baseline_snapshot_in(tx, prefix, count)) } } -fn assert_snapshot_count_in( +pub(super) fn insert_baseline_snapshot_in( tx: &Transaction<'_>, - executed_input_count: ExecutedInputCount, -) -> Result<()> { - assert_eq!( - executed_input_count, - next_executed_input_count_in(tx)?, - "snapshot executed-input count differs from canonical storage history" - ); - Ok(()) + prefix: &Path, + count: ExecutedInputCount, +) -> Result { + insert_snapshot_in(tx, prefix, None, count) } -fn insert_finalized_dump_in( +pub(super) fn insert_batch_snapshot_in( tx: &Transaction<'_>, prefix: &Path, - inclusion_block: u64, - l2_tx_index: u64, - executed_input_count: ExecutedInputCount, + batch_index: u64, + count: ExecutedInputCount, ) -> Result { - assert_snapshot_count_in(tx, executed_input_count)?; - tx.execute( - "INSERT INTO dumps (prefix) VALUES (?1)", - params![path_to_text(prefix)], - )?; - let dump_id = tx.last_insert_rowid(); - tx.execute( - "INSERT INTO finalized_snapshot \ - (singleton_id, dump_id, inclusion_block, l2_tx_index, executed_input_count) \ - VALUES (0, ?1, ?2, ?3, ?4)", - params![ - dump_id, - u64_to_i64(inclusion_block), - u64_to_i64(l2_tx_index), - u64_to_i64(executed_input_count.get()), - ], + // Registration is composed with sealing, so the batch must already be closed. + tx.query_row( + "SELECT batch_index FROM valid_closed_batches WHERE batch_index = ?1", + [u64_to_i64(batch_index)], + |_| Ok(()), )?; - Ok(dump_id) + insert_snapshot_in(tx, prefix, Some(batch_index), count) } -// ── transaction-scoped helpers ──────────────────────────────────────────── - -pub(super) fn insert_pending_dump_in( +fn insert_snapshot_in( tx: &Transaction<'_>, prefix: &Path, - nonce: u64, - l2_tx_index: u64, - executed_input_count: ExecutedInputCount, + batch_index: Option, + count: ExecutedInputCount, ) -> Result { - assert_snapshot_count_in(tx, executed_input_count)?; + assert_eq!( + count, + next_executed_input_count_in(tx)?, + "snapshot count differs from canonical storage history" + ); tx.execute( "INSERT INTO dumps (prefix) VALUES (?1)", - params![path_to_text(prefix)], + [prefix.to_string_lossy().as_ref()], )?; - let dump_id = tx.last_insert_rowid(); + let id = tx.last_insert_rowid(); tx.execute( - "INSERT INTO pending_snapshots \ - (nonce, dump_id, l2_tx_index, executed_input_count) \ - VALUES (?1, ?2, ?3, ?4)", - params![ - u64_to_i64(nonce), - dump_id, - u64_to_i64(l2_tx_index), - u64_to_i64(executed_input_count.get()), - ], + "INSERT INTO snapshots (dump_id, batch_index, executed_input_count) VALUES (?1, ?2, ?3)", + params![id, batch_index.map(u64_to_i64), u64_to_i64(count.get())], )?; - Ok(dump_id) + Ok(id) } -pub(super) fn promote_finalized_in( - tx: &Transaction<'_>, - max_nonce: u64, - inclusion_block: u64, -) -> Result<()> { - // The promoted dump's bytes correspond to state at batch close, so - // we carry over its `l2_tx_index` directly. - let (new_dump_id, l2_tx_index, executed_input_count): (i64, i64, i64) = tx.query_row( - "SELECT dump_id, l2_tx_index, executed_input_count \ - FROM pending_snapshots WHERE nonce = ?1", - params![u64_to_i64(max_nonce)], - |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), - )?; - - tx.execute( - "INSERT OR REPLACE INTO finalized_snapshot \ - (singleton_id, dump_id, inclusion_block, l2_tx_index, executed_input_count) \ - VALUES (0, ?1, ?2, ?3, ?4)", - params![ - new_dump_id, - u64_to_i64(inclusion_block), - l2_tx_index, - executed_input_count, - ], - )?; - - // Clean up the promoted row plus any stale ones behind it. The - // dumps for non-max nonces (and the previous finalized) become GC - // candidates from this point. - tx.execute( - "DELETE FROM pending_snapshots WHERE nonce <= ?1", - params![u64_to_i64(max_nonce)], - )?; - - Ok(()) +/// Required by recovery/admission inside their existing transaction. +pub(super) fn has_rollback_safe_snapshot_in(conn: &Connection) -> Result { + Ok(rollback_snapshot_in(conn)?.is_some()) } -fn acquire_dump_lease_in(tx: &Transaction<'_>, dump_id: i64) -> Result<()> { - let changed = tx.execute( - "UPDATE dumps SET lease_count = lease_count + 1 WHERE id = ?1", - params![dump_id], - )?; - if changed != 1 { - return Err(rusqlite::Error::StatementChangedRows(changed)); - } - Ok(()) -} - -fn release_dump_lease_in(tx: &Transaction<'_>, dump_id: i64) -> Result<()> { - let changed = tx.execute( - "UPDATE dumps SET lease_count = lease_count - 1 WHERE id = ?1", - params![dump_id], - )?; - if changed != 1 { - return Err(rusqlite::Error::StatementChangedRows(changed)); - } - Ok(()) -} - -fn reset_dump_leases_in(tx: &Transaction<'_>) -> Result { - tx.execute("UPDATE dumps SET lease_count = 0", []) -} - -fn gc_dump_rows_in(tx: &Transaction<'_>) -> Result> { - let mut stmt = tx.prepare( - "SELECT id, prefix FROM dumps \ - WHERE lease_count = 0 \ - AND id NOT IN (SELECT dump_id FROM pending_snapshots) \ - AND id NOT IN (SELECT dump_id FROM finalized_snapshot) \ - ORDER BY id", - )?; - let rows: Result> = stmt.query_map([], row_to_dump_row)?.collect(); - rows -} - -#[cfg(test)] -fn delete_dump_row_in(tx: &Transaction<'_>, dump_id: i64) -> Result<()> { - let changed = tx.execute("DELETE FROM dumps WHERE id = ?1", params![dump_id])?; - if changed != 1 { - return Err(rusqlite::Error::StatementChangedRows(changed)); +fn rollback_snapshot_in(conn: &Connection) -> Result> { + match latest_accepted_boundary_in(conn)? { + Some((batch_index, _, _)) => snapshot_for_batch_in(conn, batch_index).map(Some), + None => baseline_snapshot_in(conn), } - Ok(()) } -fn latest_pending_dump_in(tx: &Transaction<'_>) -> Result> { - tx.query_row( - "SELECT p.nonce, p.dump_id, d.prefix, p.l2_tx_index, p.executed_input_count \ - FROM pending_snapshots p \ - LEFT JOIN dumps d ON d.id = p.dump_id \ - ORDER BY p.nonce DESC \ - LIMIT 1", +fn latest_accepted_boundary_in(conn: &Connection) -> Result> { + conn.query_row( + "SELECT b.batch_index, a.nonce, a.inclusion_block FROM safe_accepted_batches a \ + LEFT JOIN valid_closed_batches b ON b.nonce = a.nonce \ + ORDER BY a.safe_input_index DESC LIMIT 1", [], |row| { - let nonce: i64 = row.get(0)?; - let dump_id: i64 = row.get(1)?; - // LEFT JOIN keeps a dangling reference visible; reading NULL as - // String then returns InvalidColumnType instead of laundering the - // corruption into OptionalExtension's `None`. - let prefix: String = row.get(2)?; - let l2_tx_index: i64 = row.get(3)?; - let executed_input_count: i64 = row.get(4)?; - Ok(PendingDump { - nonce: i64_to_u64(nonce), - dump: DumpRow { - id: dump_id, - prefix: PathBuf::from(prefix), - }, - l2_tx_index: i64_to_u64(l2_tx_index), - executed_input_count: ExecutedInputCount::new(i64_to_u64(executed_input_count)), - }) + Ok(( + i64_to_u64(row.get(0)?), + i64_to_u64(row.get(1)?), + i64_to_u64(row.get(2)?), + )) }, ) .optional() } -fn finalized_dump_in(tx: &Transaction<'_>) -> Result> { - tx.query_row( - "SELECT f.dump_id, d.prefix, f.inclusion_block, f.l2_tx_index, \ - f.executed_input_count \ - FROM finalized_snapshot f \ - LEFT JOIN dumps d ON d.id = f.dump_id \ - WHERE f.singleton_id = 0", +fn snapshot_for_batch_in(conn: &Connection, batch_index: u64) -> Result { + conn.query_row( + "SELECT s.dump_id, d.prefix, s.executed_input_count FROM snapshots s \ + LEFT JOIN dumps d ON d.id = s.dump_id WHERE s.batch_index = ?1", + [u64_to_i64(batch_index)], + row_to_snapshot, + ) +} + +fn baseline_snapshot_in(conn: &Connection) -> Result> { + conn.query_row( + "SELECT s.dump_id, d.prefix, s.executed_input_count FROM snapshots s \ + LEFT JOIN dumps d ON d.id = s.dump_id WHERE s.batch_index IS NULL", [], - |row| { - let dump_id: i64 = row.get(0)?; - // See latest_pending_dump_in: a missing referenced dump row must - // be an error, not an apparent absence of the singleton. - let prefix: String = row.get(1)?; - let inclusion_block: i64 = row.get(2)?; - let l2_tx_index: i64 = row.get(3)?; - let executed_input_count: i64 = row.get(4)?; - Ok(FinalizedDump { - dump: DumpRow { - id: dump_id, - prefix: PathBuf::from(prefix), - }, - inclusion_block: i64_to_u64(inclusion_block), - l2_tx_index: i64_to_u64(l2_tx_index), - executed_input_count: ExecutedInputCount::new(i64_to_u64(executed_input_count)), - }) - }, + row_to_snapshot, ) .optional() } -/// Select the snapshot to resume or serve from: the latest pending dump, else -/// the finalized snapshot. Shared by [`Storage::latest_snapshot`] (catch-up's -/// resume checkpoint) and [`Storage::acquire_latest_snapshot_lease`] (the -/// `/latest_snapshot` lease), so the "pending else finalized" rule lives in one -/// place. -fn latest_snapshot_in(tx: &Transaction<'_>) -> Result> { - Ok(match latest_pending_dump_in(tx)? { - Some(pending) => Some(( - pending.dump, - pending.l2_tx_index, - pending.executed_input_count, - )), - None => finalized_dump_in(tx)?.map(|f| (f.dump, f.l2_tx_index, f.executed_input_count)), - }) +fn finalized_dump_in(conn: &Connection) -> Result> { + if let Some((batch_index, nonce, inclusion_block)) = latest_accepted_boundary_in(conn)? { + let snapshot = snapshot_for_batch_in(conn, batch_index)?; + return Ok(Some(FinalizedDump { + dump: snapshot.dump, + executed_input_count: snapshot.executed_input_count, + inclusion_block, + next_batch_nonce: nonce.checked_add(1).expect("accepted nonce overflow"), + })); + } + // Genesis is independently known canonical. A rebuilt baseline may contain + // the recovery fold's speculative final drain and is never comparable at C. + let Some(snapshot) = baseline_snapshot_in(conn)? else { + return Ok(None); + }; + let history = query_history_state(conn)?; + if history.base_safe_block == 0 && history.base_executed_input_count == 0 { + Ok(Some(FinalizedDump { + dump: snapshot.dump, + executed_input_count: snapshot.executed_input_count, + inclusion_block: 0, + next_batch_nonce: super::mutations::batch_tree_anchor_in(conn)?, + })) + } else { + Ok(None) + } } -fn list_dump_rows_in(tx: &Transaction<'_>) -> Result> { - let mut stmt = tx.prepare("SELECT id, prefix FROM dumps ORDER BY id")?; - let rows: Result> = stmt.query_map([], row_to_dump_row)?.collect(); - rows +fn latest_snapshot_in(conn: &Connection) -> Result> { + let batch_index: Option = conn + .query_row( + "SELECT batch_index FROM valid_closed_batches ORDER BY batch_index DESC LIMIT 1", + [], + |r| r.get(0), + ) + .optional()?; + match batch_index { + Some(index) => snapshot_for_batch_in(conn, i64_to_u64(index)).map(Some), + None => baseline_snapshot_in(conn), + } } -/// Visible to siblings within the `storage` module so that the -/// danger-zone recovery path can compose pending-dump cleanup into -/// the same transaction as `cascade_invalidate_from` — otherwise a -/// crash between the cascade and the clear would leave stale pending -/// snapshots pointing at states the canonical stream will never reach. -#[cfg(test)] -pub(super) fn clear_pending_dumps_in(tx: &Transaction<'_>) -> Result { - tx.execute("DELETE FROM pending_snapshots", []) +fn snapshot_next_nonce_in(conn: &Connection, dump_id: i64) -> Result { + let batch_index: Option = conn.query_row( + "SELECT batch_index FROM snapshots WHERE dump_id = ?1", + [dump_id], + |r| r.get(0), + )?; + match batch_index { + Some(index) => Ok(batch_nonce_in(conn, i64_to_u64(index))? + .checked_add(1) + .expect("batch nonce overflow")), + None => super::mutations::batch_tree_anchor_in(conn), + } } -/// Scoped pending-snapshot clear for the recovery cascade: delete only the -/// rows whose `nonce >= from_nonce` (the cascade pivot's nonce) — exactly -/// the cascaded batches' pendings. Lower-nonce rows are gold-but-unpromoted -/// pendings that must survive (deleting them arms a -/// promote-wedge crash-loop when their landing is later observed). Same -/// same-transaction composition rationale as [`clear_pending_dumps_in`]. -pub(super) fn clear_pending_dumps_from_nonce_in( - tx: &Transaction<'_>, - from_nonce: u64, -) -> Result { - tx.execute( - "DELETE FROM pending_snapshots WHERE nonce >= ?1", - params![u64_to_i64(from_nonce)], - ) +fn acquire_dump_lease_in(tx: &Transaction<'_>, id: i64) -> Result<()> { + let changed = tx.execute( + "UPDATE dumps SET lease_count = lease_count + 1 WHERE id = ?1", + [id], + )?; + if changed != 1 { + return Err(rusqlite::Error::StatementChangedRows(changed)); + } + Ok(()) } -/// Free-function form of [`Storage::batch_nonce`] for composing into a -/// larger transaction (the recovery cascade reads its pivot's nonce to -/// scope the pending clear). -pub(super) fn batch_nonce_in(conn: &rusqlite::Connection, batch_index: u64) -> Result { - let nonce: i64 = conn.query_row( - "SELECT nonce FROM batches WHERE batch_index = ?1", - params![u64_to_i64(batch_index)], - |row| row.get(0), +fn release_dump_lease_in(tx: &Transaction<'_>, id: i64) -> Result<()> { + let changed = tx.execute( + "UPDATE dumps SET lease_count = lease_count - 1 WHERE id = ?1", + [id], )?; - Ok(i64_to_u64(nonce)) + if changed != 1 { + return Err(rusqlite::Error::StatementChangedRows(changed)); + } + Ok(()) +} + +pub(super) fn batch_nonce_in(conn: &Connection, batch_index: u64) -> Result { + conn.query_row( + "SELECT nonce FROM batches WHERE batch_index = ?1", + [u64_to_i64(batch_index)], + |row| Ok(i64_to_u64(row.get(0)?)), + ) } fn row_to_dump_row(row: &rusqlite::Row<'_>) -> Result { @@ -769,630 +446,162 @@ fn row_to_dump_row(row: &rusqlite::Row<'_>) -> Result { }) } -fn path_to_text(path: &Path) -> String { - // Prefixes are produced by the lane and are valid UTF-8 by - // construction (they're built from u64 ids and unicode-safe - // configured data dirs). `to_string_lossy` is the well-trodden - // path for crossing the Path → SQLite TEXT boundary; if the lane - // ever produces a non-UTF8 prefix that's a separate bug we'd - // catch via the round-trip read mismatch. - path.to_string_lossy().into_owned() +fn row_to_snapshot(row: &rusqlite::Row<'_>) -> Result { + Ok(Snapshot { + dump: row_to_dump_row(row)?, + executed_input_count: ExecutedInputCount::new(i64_to_u64(row.get(2)?)), + }) } #[cfg(test)] mod tests { + use super::*; + use crate::ingress::inclusion_lane::{IncludedUserOp, PendingUserOp}; + use crate::storage::history::{advance_recovery_generation_in, initialize_history_in}; + use crate::storage::test_helpers::{ + SENDER_A, pin_test_deployment_identity, seed_safe_inputs_with_batch_nonces, temp_db, + }; + use crate::storage::{LifecycleCommand, SafeInputRange, WriteHead}; use alloy_primitives::{Address, Signature}; - use sequencer_core::history::RecoveryGeneration; use sequencer_core::user_op::{SignedUserOp, UserOp}; - use std::collections::HashSet; - use std::path::PathBuf; - use std::sync::Arc; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::time::SystemTime; - use tokio::sync::oneshot; - - use crate::ingress::inclusion_lane::{IncludedUserOp, PendingUserOp}; - use crate::storage::{ - ExecutedInputCount, LifecycleCommand, SafeInputRange, Storage, - history::advance_recovery_generation_in, test_helpers::temp_db, - }; - use super::{DumpRow, FinalizedDump, FinalizedLease, LeaseGuard, PendingDump}; - - fn prefix(n: u64) -> PathBuf { - PathBuf::from(format!("/data/dumps/{n}")) + fn inline(release: Box) { + release(); } - - #[test] - fn insert_pending_creates_dump_and_pending_rows() { - let db = temp_db("insert-pending"); - let mut storage = Storage::open(db.path.as_str()).expect("open"); - - let dump_id = storage - .insert_pending_dump(&prefix(0), 0, 10) - .expect("insert"); - - let pending = storage.latest_pending_dump().expect("read").expect("some"); - assert_eq!( - pending, - PendingDump { - nonce: 0, - dump: DumpRow { - id: dump_id, - prefix: prefix(0), - }, - l2_tx_index: 10, - executed_input_count: ExecutedInputCount::ZERO, - } - ); - - let rows = storage.list_dump_rows().expect("list"); - assert_eq!(rows.len(), 1); - assert_eq!(rows[0].prefix, prefix(0)); + fn reporter() -> PersistentReleaseFailureReporter { + Arc::new(|_| {}) } - - #[test] - fn insert_pending_rejects_duplicate_prefix() { - let db = temp_db("dup-prefix"); - let mut storage = Storage::open(db.path.as_str()).expect("open"); - - storage - .insert_pending_dump(&prefix(0), 0, 0) - .expect("first"); - let err = storage - .insert_pending_dump(&prefix(0), 1, 0) - .expect_err("second"); - assert!( - err.to_string().contains("UNIQUE"), - "expected UNIQUE failure, got: {err}" - ); + fn prefix(n: u64) -> PathBuf { + PathBuf::from(format!("/snapshot/{n}")) } - #[test] - fn initial_finalized_snapshot_binds_rebuild_base_atomically() { - let db = temp_db("initial-finalized-history-base"); - let mut storage = - Storage::initialize_for_command(db.path.as_str(), LifecycleCommand::Rebuild) - .expect("initialize rebuild"); - assert_eq!( - storage - .history_state() - .expect("pending history") - .base_executed_input_count, - None - ); - assert_eq!( - storage - .history_state() - .expect("pending history") - .base_safe_input_index, - None - ); - + fn close(storage: &mut Storage, head: &mut WriteHead) -> i64 { + let index = head.batch_index; + let safe_block = head.safe_block; + let count = storage.next_executed_input_count().unwrap(); storage - .conn - .execute_batch( - "CREATE TRIGGER fail_initial_finalized_snapshot - BEFORE INSERT ON finalized_snapshot - BEGIN - SELECT RAISE(ABORT, 'injected finalized snapshot failure'); - END;", - ) - .expect("install failure trigger"); - let err = storage - .insert_initial_finalized_dump(&prefix(0), 100, 7, 41, 7) - .expect_err("snapshot failure must roll back the history base"); - assert!( - err.to_string() - .contains("injected finalized snapshot failure"), - "unexpected error: {err:?}" - ); - assert_eq!( - storage - .history_state() - .expect("history after rollback") - .base_executed_input_count, - None, - "the base cannot survive without its establishing snapshot" - ); - assert_eq!( - storage - .history_state() - .expect("history after rollback") - .base_safe_input_index, - None, - "the safe-input floor cannot survive without its establishing snapshot" - ); - assert!(storage.finalized_dump().expect("read finalized").is_none()); - assert!(storage.list_dump_rows().expect("read dumps").is_empty()); - - storage - .conn - .execute_batch("DROP TRIGGER fail_initial_finalized_snapshot;") - .expect("remove failure trigger"); - storage - .insert_initial_finalized_dump(&prefix(1), 100, 7, 41, 7) - .expect("bind base with finalized snapshot"); - assert_eq!( - storage - .history_state() - .expect("bound history") - .base_executed_input_count, - Some(41) - ); - assert_eq!( - storage - .history_state() - .expect("bound history") - .base_safe_input_index, - Some(7) - ); - assert_eq!( - storage - .finalized_dump() - .expect("read finalized") - .expect("finalized snapshot") - .l2_tx_index, - 7, - "the physical replay cursor remains distinct from application base K" - ); - } - - #[test] - fn latest_pending_picks_highest_nonce() { - let db = temp_db("latest-pending"); - let mut storage = Storage::open(db.path.as_str()).expect("open"); - - storage.insert_pending_dump(&prefix(0), 0, 100).unwrap(); - storage.insert_pending_dump(&prefix(1), 2, 102).unwrap(); - storage.insert_pending_dump(&prefix(2), 1, 101).unwrap(); - - let latest = storage.latest_pending_dump().unwrap().unwrap(); - assert_eq!(latest.nonce, 2); - assert_eq!(latest.l2_tx_index, 102); - assert_eq!(latest.dump.prefix, prefix(1)); + .close_frame_and_batch_with_snapshot(head, safe_block, &prefix(index + 1), index, count) + .unwrap(); + storage.latest_snapshot().unwrap().unwrap().dump.id } #[test] - fn promote_moves_max_nonce_to_finalized_and_clears_promoted_pending() { - let db = temp_db("promote"); - let mut storage = Storage::open(db.path.as_str()).expect("open"); - - let id_a = storage.insert_pending_dump(&prefix(0), 0, 100).unwrap(); - let id_b = storage.insert_pending_dump(&prefix(1), 1, 101).unwrap(); - let id_c = storage.insert_pending_dump(&prefix(2), 2, 102).unwrap(); - let id_unrelated = storage.insert_pending_dump(&prefix(3), 3, 103).unwrap(); - - storage.promote_finalized(2, 500).unwrap(); - - // Finalized points at the dump for nonce 2. - let finalized = storage.finalized_dump().unwrap().unwrap(); + fn acceptance_derives_checkpoint_and_gc_preserves_every_future_candidate() { + let db = temp_db("derived-snapshot-gc"); + let mut storage = Storage::open(&db.path).unwrap(); + pin_test_deployment_identity(&mut storage, SENDER_A); + let baseline = storage + .insert_baseline_snapshot(&prefix(0), ExecutedInputCount::ZERO) + .unwrap(); + let mut head = storage + .initialize_open_state(0, SafeInputRange::empty_at(0)) + .unwrap(); + let first = close(&mut storage, &mut head); + let second = close(&mut storage, &mut head); + let third = close(&mut storage, &mut head); + assert!(storage.gc_unreferenced_dumps().unwrap().is_empty()); + assert_eq!(storage.finalized_dump().unwrap().unwrap().dump.id, baseline); + seed_safe_inputs_with_batch_nonces(&mut storage, SENDER_A, 1, &[0]); + assert_eq!(storage.finalized_dump().unwrap().unwrap().dump.id, first); assert_eq!( - finalized, - FinalizedDump { - dump: DumpRow { - id: id_c, - prefix: prefix(2), - }, - inclusion_block: 500, - l2_tx_index: 102, - executed_input_count: ExecutedInputCount::ZERO, - } - ); - - // Nonces 0, 1, 2 are gone from pending; 3 stays. - let latest = storage.latest_pending_dump().unwrap().unwrap(); - assert_eq!(latest.nonce, 3); - assert_eq!(latest.dump.id, id_unrelated); - - // Dumps 0 and 1 are now GC-eligible (unreferenced); 2 is in - // finalized and 3 is in pending, so both still referenced. - let gc: HashSet = storage - .gc_dump_rows() - .unwrap() - .into_iter() - .map(|row| row.id) - .collect(); - assert_eq!(gc, HashSet::from([id_a, id_b])); - } - - #[test] - fn promote_overwrites_previous_finalized_and_makes_old_dump_gc_eligible() { - let db = temp_db("promote-overwrite"); - let mut storage = Storage::open(db.path.as_str()).expect("open"); - - let id_first = storage.insert_pending_dump(&prefix(0), 0, 100).unwrap(); - storage.promote_finalized(0, 500).unwrap(); - - let id_second = storage.insert_pending_dump(&prefix(1), 1, 101).unwrap(); - storage.promote_finalized(1, 501).unwrap(); - - let finalized = storage.finalized_dump().unwrap().unwrap(); - assert_eq!(finalized.dump.id, id_second); - - let gc: HashSet = storage - .gc_dump_rows() - .unwrap() - .into_iter() - .map(|row| row.id) - .collect(); - assert_eq!(gc, HashSet::from([id_first])); - } - - #[test] - fn lease_acquire_release_round_trips_and_blocks_gc() { - let db = temp_db("lease-roundtrip"); - let mut storage = Storage::open(db.path.as_str()).expect("open"); - - let dump_id = storage.insert_pending_dump(&prefix(0), 0, 0).unwrap(); - storage.promote_finalized(0, 500).unwrap(); - - // Promote another so the first becomes a GC candidate. - let _ = storage.insert_pending_dump(&prefix(1), 1, 1).unwrap(); - storage.promote_finalized(1, 501).unwrap(); - - // Without a lease the first dump is GC-eligible. - assert!( - storage - .gc_dump_rows() - .unwrap() - .iter() - .any(|row| row.id == dump_id) - ); - - // Acquire the lease and the dump is no longer eligible. - storage.acquire_dump_lease(dump_id).unwrap(); - assert!( storage - .gc_dump_rows() + .gc_unreferenced_dumps() .unwrap() .iter() - .all(|row| row.id != dump_id) - ); - - // Release and it's eligible again. - storage.release_dump_lease(dump_id).unwrap(); - assert!( - storage - .gc_dump_rows() - .unwrap() - .iter() - .any(|row| row.id == dump_id) - ); - } - - #[test] - fn lease_supports_multiple_holders() { - let db = temp_db("lease-multi"); - let mut storage = Storage::open(db.path.as_str()).expect("open"); - - let dump_id = storage.insert_pending_dump(&prefix(0), 0, 0).unwrap(); - storage.promote_finalized(0, 0).unwrap(); - let _ = storage.insert_pending_dump(&prefix(1), 1, 1).unwrap(); - storage.promote_finalized(1, 1).unwrap(); - - // Two concurrent readers. - storage.acquire_dump_lease(dump_id).unwrap(); - storage.acquire_dump_lease(dump_id).unwrap(); - - // One releases — still leased. - storage.release_dump_lease(dump_id).unwrap(); - assert!( - storage - .gc_dump_rows() - .unwrap() - .iter() - .all(|row| row.id != dump_id) - ); - - // Second releases — now eligible. - storage.release_dump_lease(dump_id).unwrap(); - assert!( - storage - .gc_dump_rows() - .unwrap() - .iter() - .any(|row| row.id == dump_id) - ); - } - - #[test] - fn release_below_zero_is_rejected_by_check_constraint() { - let db = temp_db("lease-underflow"); - let mut storage = Storage::open(db.path.as_str()).expect("open"); - - let dump_id = storage.insert_pending_dump(&prefix(0), 0, 0).unwrap(); - let err = storage - .release_dump_lease(dump_id) - .expect_err("lease_count cannot go negative"); - assert!( - err.to_string().contains("CHECK"), - "expected CHECK constraint failure, got: {err}" - ); - } - - #[test] - fn reset_dump_leases_clears_every_lease() { - let db = temp_db("lease-reset"); - let mut storage = Storage::open(db.path.as_str()).expect("open"); - - let id_a = storage.insert_pending_dump(&prefix(0), 0, 0).unwrap(); - let id_b = storage.insert_pending_dump(&prefix(1), 1, 0).unwrap(); - storage.acquire_dump_lease(id_a).unwrap(); - storage.acquire_dump_lease(id_a).unwrap(); - storage.acquire_dump_lease(id_b).unwrap(); - - let cleared = storage.reset_dump_leases().unwrap(); - assert_eq!(cleared, 2); - - // After reset, releasing would underflow if leases lingered; - // since GC eligibility depends on lease_count == 0 AND no - // references, both rows still have pending references and - // aren't GC-eligible — but a follow-up promote would correctly - // surface them. - storage.promote_finalized(1, 0).unwrap(); - // id_a's dump is now superseded; id_b is finalized. - let gc: Vec = storage - .gc_dump_rows() - .unwrap() - .into_iter() - .map(|row| row.id) - .collect(); - assert_eq!(gc, vec![id_a]); - } - - #[test] - fn delete_dump_row_removes_from_dumps_table() { - let db = temp_db("delete-row"); - let mut storage = Storage::open(db.path.as_str()).expect("open"); - - let dump_id = storage.insert_pending_dump(&prefix(0), 0, 0).unwrap(); - storage.promote_finalized(0, 0).unwrap(); - - // While in finalized, delete should fail (FK restrict). - let err = storage - .delete_dump_row(dump_id) - .expect_err("FK should prevent delete while finalized"); - assert!( - err.to_string().contains("FOREIGN KEY"), - "expected FK failure, got: {err}" + .map(|r| r.id) + .collect::>(), + vec![baseline] ); - - // Promote a successor so finalized no longer references the row. - let _ = storage.insert_pending_dump(&prefix(1), 1, 0).unwrap(); - storage.promote_finalized(1, 0).unwrap(); - - storage.delete_dump_row(dump_id).expect("delete after gc"); - assert!( + seed_safe_inputs_with_batch_nonces(&mut storage, SENDER_A, 2, &[1]); + assert_eq!(storage.finalized_dump().unwrap().unwrap().dump.id, second); + assert_eq!(storage.latest_snapshot().unwrap().unwrap().dump.id, third); + assert_eq!( storage - .list_dump_rows() + .gc_unreferenced_dumps() .unwrap() .iter() - .all(|row| row.id != dump_id) + .map(|r| r.id) + .collect::>(), + vec![first] ); } #[test] - fn clear_pending_removes_all_pending_rows() { - let db = temp_db("clear-pending"); - let mut storage = Storage::open(db.path.as_str()).expect("open"); - - let _id_a = storage.insert_pending_dump(&prefix(0), 0, 0).unwrap(); - let id_b = storage.insert_pending_dump(&prefix(1), 1, 0).unwrap(); - // Move id_a into finalized. - storage.promote_finalized(0, 0).unwrap(); - // id_b stays in pending. - - let cleared = storage.clear_pending_dumps().unwrap(); - assert_eq!(cleared, 1); - - assert!(storage.latest_pending_dump().unwrap().is_none()); - - // id_b is now unreferenced → GC eligible. id_a stays in finalized. - let gc: Vec = storage - .gc_dump_rows() - .unwrap() - .into_iter() - .map(|row| row.id) - .collect(); - assert_eq!(gc, vec![id_b]); - } - - #[test] - fn list_dump_rows_returns_everything() { - let db = temp_db("list-rows"); - let mut storage = Storage::open(db.path.as_str()).expect("open"); - - storage.insert_pending_dump(&prefix(0), 0, 0).unwrap(); - storage.insert_pending_dump(&prefix(1), 1, 0).unwrap(); - storage.insert_pending_dump(&prefix(2), 2, 0).unwrap(); - storage.promote_finalized(0, 0).unwrap(); - - let rows = storage.list_dump_rows().unwrap(); - assert_eq!(rows.len(), 3); - let prefixes: HashSet = rows.into_iter().map(|row| row.prefix).collect(); - assert_eq!(prefixes, HashSet::from([prefix(0), prefix(1), prefix(2)])); - } - - #[test] - fn promote_missing_nonce_errors() { - let db = temp_db("promote-missing"); - let mut storage = Storage::open(db.path.as_str()).expect("open"); - - let err = storage - .promote_finalized(42, 0) - .expect_err("missing nonce should error"); - assert!(matches!(err, rusqlite::Error::QueryReturnedNoRows)); - } - - #[test] - fn gc_unreferenced_dumps_drops_rows_and_returns_them() { - let db = temp_db("gc-unreferenced"); - let mut storage = Storage::open(db.path.as_str()).expect("open"); - - // Set up: 4 dumps. ids 0, 1 superseded (unreferenced after - // promotion). id 2 in finalized. id 3 in pending. - let _id_a = storage.insert_pending_dump(&prefix(0), 0, 0).unwrap(); - storage.promote_finalized(0, 0).unwrap(); - let _id_b = storage.insert_pending_dump(&prefix(1), 1, 0).unwrap(); - storage.promote_finalized(1, 0).unwrap(); - // Now id_a is superseded by id_b's promotion. id_a is GC-eligible. - let _id_c = storage.insert_pending_dump(&prefix(2), 2, 0).unwrap(); - storage.promote_finalized(2, 0).unwrap(); - let _id_d = storage.insert_pending_dump(&prefix(3), 3, 0).unwrap(); - - let removed = storage.gc_unreferenced_dumps().unwrap(); - let removed_ids: HashSet = removed.iter().map(|row| row.id).collect(); - - // The two superseded dumps got removed. The current finalized - // (id_c) and the pending (id_d) survived. - assert_eq!(removed.len(), 2); - assert!(!removed_ids.is_empty()); - - // Confirm the survivors are still in dumps. - let surviving: HashSet = storage - .list_dump_rows() - .unwrap() - .into_iter() - .map(|row| row.prefix) - .collect(); - assert!(surviving.contains(&prefix(2))); - assert!(surviving.contains(&prefix(3))); - assert!(!surviving.contains(&prefix(0))); - assert!(!surviving.contains(&prefix(1))); - } - - #[test] - fn gc_unreferenced_dumps_skips_leased_rows() { - let db = temp_db("gc-leased"); - let mut storage = Storage::open(db.path.as_str()).expect("open"); - - // Two dumps that would both be GC-eligible… - let id_a = storage.insert_pending_dump(&prefix(0), 0, 0).unwrap(); - storage.promote_finalized(0, 0).unwrap(); - let id_b = storage.insert_pending_dump(&prefix(1), 1, 0).unwrap(); - storage.promote_finalized(1, 0).unwrap(); - // id_a is superseded. Both ids might be candidates; the - // promotion of 1 makes id_a unreferenced and id_b finalized. - - // … but we hold a lease on id_a, so GC must skip it. - storage.acquire_dump_lease(id_a).unwrap(); - - let removed = storage.gc_unreferenced_dumps().unwrap(); - let removed_ids: HashSet = removed.iter().map(|row| row.id).collect(); - assert!(!removed_ids.contains(&id_a), "lease blocks GC"); - assert!(!removed_ids.contains(&id_b), "id_b is still finalized"); - - // Releasing the lease makes id_a eligible. - storage.release_dump_lease(id_a).unwrap(); - let removed_again = storage.gc_unreferenced_dumps().unwrap(); - let removed_again_ids: HashSet = removed_again.iter().map(|row| row.id).collect(); - assert!(removed_again_ids.contains(&id_a)); - } - - #[test] - fn gc_unreferenced_dumps_on_empty_db_returns_empty() { - let db = temp_db("gc-empty"); - let mut storage = Storage::open(db.path.as_str()).expect("open"); - let removed = storage.gc_unreferenced_dumps().unwrap(); - assert!(removed.is_empty()); - } - - /// Release scheduler for tests: run the release inline (no async runtime to - /// offload to). The lease guard's `Drop` hands its release closure here. - fn inline(release: Box) { - release(); - } - - fn install_deferred_lease_commit_failure(path: &str) { - let conn = Storage::open_connection(path).expect("open failure-injection connection"); - conn.execute_batch( - "CREATE TABLE lease_commit_parent ( - id INTEGER PRIMARY KEY - ); - CREATE TABLE lease_commit_child ( - id INTEGER PRIMARY KEY, - parent_id INTEGER NOT NULL - REFERENCES lease_commit_parent(id) - DEFERRABLE INITIALLY DEFERRED - ); - CREATE TRIGGER fail_lease_commit - AFTER UPDATE OF lease_count ON dumps - WHEN NEW.lease_count > OLD.lease_count - BEGIN - INSERT INTO lease_commit_child(parent_id) VALUES (1); - END;", - ) - .expect("install deferred commit failure"); - } - - fn noop_reporter() -> super::PersistentReleaseFailureReporter { - Arc::new(|_cause: &str| {}) - } - - fn counting_scheduler(scheduled: Arc) -> super::ReleaseScheduler { - Arc::new(move |_release| { - scheduled.fetch_add(1, Ordering::SeqCst); - }) + fn missing_exact_accepted_snapshot_never_falls_back_or_collects() { + let db = temp_db("missing-accepted-snapshot"); + let mut storage = Storage::open(&db.path).unwrap(); + pin_test_deployment_identity(&mut storage, SENDER_A); + storage + .insert_baseline_snapshot(&prefix(0), ExecutedInputCount::ZERO) + .unwrap(); + let mut head = storage + .initialize_open_state(0, SafeInputRange::empty_at(0)) + .unwrap(); + close(&mut storage, &mut head); + close(&mut storage, &mut head); + seed_safe_inputs_with_batch_nonces(&mut storage, SENDER_A, 1, &[0, 1]); + storage + .conn + .execute("DELETE FROM snapshots WHERE batch_index=1", []) + .unwrap(); + assert!(matches!( + storage.finalized_dump(), + Err(rusqlite::Error::QueryReturnedNoRows) + )); + assert!(matches!( + storage.latest_snapshot(), + Err(rusqlite::Error::QueryReturnedNoRows) + )); + assert!(storage.has_rollback_safe_snapshot().is_err()); + assert!(storage.gc_unreferenced_dumps().is_err()); + assert_eq!(storage.list_dump_rows().unwrap().len(), 3); } #[test] - fn lease_release_reports_persistent_missing_row_failure() { - let db = temp_db("lease-release-persistent-failure"); - let _storage = Storage::open(db.path.as_str()).expect("open"); - let reported = Arc::new(AtomicBool::new(false)); - let reporter = { - let reported = reported.clone(); - Arc::new(move |_cause: &str| { - reported.store(true, Ordering::SeqCst); + fn rebuilt_baseline_is_restorable_but_not_an_accepted_comparison_checkpoint() { + let db = temp_db("rebuilt-snapshot"); + let mut storage = + Storage::initialize_for_command(&db.path, LifecycleCommand::Rebuild).unwrap(); + storage + .write(|tx| { + initialize_history_in(tx, ExecutedInputCount::new(41), 100)?; + insert_baseline_snapshot_in(tx, &prefix(0), ExecutedInputCount::new(41))?; + Ok(()) }) - }; - let guard = LeaseGuard { - path: db.path, - dump_id: i64::MAX, - schedule: Arc::new(inline), - report_persistent_failure: reporter, - }; - - drop(guard); - - assert!( - reported.load(Ordering::SeqCst), - "a durable lease-row invariant failure must reach the runtime reporter" - ); - } - - #[test] - fn acquire_finalized_lease_returns_none_when_no_finalized() { - let db = temp_db("acquire-finalized-none"); - let mut storage = Storage::open(db.path.as_str()).expect("open"); - assert!( + .unwrap(); + assert_eq!( storage - .acquire_finalized_lease(Arc::new(inline), noop_reporter()) + .latest_snapshot() + .unwrap() .unwrap() - .is_none() + .executed_input_count + .get(), + 41 ); + assert!(storage.finalized_dump().unwrap().is_none()); + assert!(storage.has_rollback_safe_snapshot().unwrap()); + assert!(storage.gc_unreferenced_dumps().unwrap().is_empty()); } #[test] - fn snapshot_leases_keep_artifact_count_and_history_after_promotion_and_generation_advance() { - let db = temp_db("snapshot-lease-history"); - let mut storage = Storage::open(db.path.as_str()).expect("open"); + fn leases_keep_artifact_count_and_version_together_after_acceptance_and_generation_change() { + let db = temp_db("snapshot-version"); + let mut storage = Storage::open(&db.path).unwrap(); + pin_test_deployment_identity(&mut storage, SENDER_A); + let baseline = storage + .insert_baseline_snapshot(&prefix(0), ExecutedInputCount::ZERO) + .unwrap(); let mut head = storage .initialize_open_state(0, SafeInputRange::empty_at(0)) .unwrap(); - let first_id = storage.insert_finalized_dump(&prefix(0), 100, 0).unwrap(); - let first_version = storage.history_state().unwrap().version; - let first_finalized = storage - .acquire_finalized_lease(Arc::new(inline), noop_reporter()) - .unwrap() - .unwrap(); - let first_latest = storage - .acquire_latest_snapshot_lease(Arc::new(inline), noop_reporter()) + let old = storage + .acquire_latest_snapshot_lease(Arc::new(inline), reporter()) .unwrap() .unwrap(); - - let (respond_to, _response) = oneshot::channel(); + let old_version = old.history_version; + let (respond_to, _) = tokio::sync::oneshot::channel(); storage .append_executed_user_ops_chunk( &mut head, @@ -1414,262 +623,131 @@ mod tests { }], ) .unwrap(); - let next_id = storage.insert_pending_dump(&prefix(1), 0, 1).unwrap(); - let pending = storage - .acquire_latest_snapshot_lease(Arc::new(inline), noop_reporter()) + close(&mut storage, &mut head); + seed_safe_inputs_with_batch_nonces(&mut storage, SENDER_A, 1, &[0]); + storage.write(advance_recovery_generation_in).unwrap(); + let new = storage + .acquire_finalized_lease(Arc::new(inline), reporter()) .unwrap() .unwrap(); - storage.promote_finalized(0, 101).unwrap(); - storage.write(advance_recovery_generation_in).unwrap(); - - let next_version = storage.history_state().unwrap().version; - assert_eq!(next_version.era_id, first_version.era_id); - assert_eq!(next_version.recovery_generation, RecoveryGeneration::new(1)); - for leased in [&first_finalized.dump, &first_latest] { - assert_eq!(leased.prefix, prefix(0)); - assert_eq!(leased.executed_input_count, ExecutedInputCount::ZERO); - assert_eq!(leased.l2_tx_index, 0); - assert_eq!(leased.history_version, first_version); - } - assert_eq!(first_finalized.inclusion_block, 100); - assert_eq!(pending.prefix, prefix(1)); - assert_eq!(pending.executed_input_count, ExecutedInputCount::new(1)); - assert_eq!(pending.history_version, first_version); + assert_eq!(old.prefix, prefix(0)); + assert_eq!(old.executed_input_count, ExecutedInputCount::ZERO); + assert_eq!(old.history_version, old_version); + assert_eq!(new.dump.executed_input_count.get(), 1); + assert_eq!(new.dump.next_batch_nonce, 1); + assert_ne!(new.dump.history_version, old_version); + assert!(storage.gc_unreferenced_dumps().unwrap().is_empty()); + drop(old); + assert_eq!(storage.gc_unreferenced_dumps().unwrap()[0].id, baseline); + } - let next_finalized = storage - .acquire_finalized_lease(Arc::new(inline), noop_reporter()) - .unwrap() + #[test] + fn invalidated_snapshot_is_not_selected_and_its_lease_blocks_collection() { + let db = temp_db("invalidated-snapshot"); + let mut storage = Storage::open(&db.path).unwrap(); + storage + .insert_baseline_snapshot(&prefix(0), ExecutedInputCount::ZERO) + .unwrap(); + let mut head = storage + .initialize_open_state(0, SafeInputRange::empty_at(0)) .unwrap(); - let next_latest = storage - .acquire_latest_snapshot_lease(Arc::new(inline), noop_reporter()) + let id = close(&mut storage, &mut head); + let lease = storage + .acquire_latest_snapshot_lease(Arc::new(inline), reporter()) .unwrap() .unwrap(); - for leased in [&next_finalized.dump, &next_latest] { - assert_eq!(leased.prefix, prefix(1)); - assert_eq!(leased.executed_input_count, ExecutedInputCount::new(1)); - assert_eq!(leased.l2_tx_index, 1); - assert_eq!(leased.history_version, next_version); - } - assert_eq!(next_finalized.inclusion_block, 101); - assert!(storage.gc_unreferenced_dumps().unwrap().is_empty()); - assert_eq!(storage.dump_lease_count(first_id).unwrap(), Some(2)); - assert_eq!(storage.dump_lease_count(next_id).unwrap(), Some(3)); - - drop(( - first_finalized, - first_latest, - pending, - next_finalized, - next_latest, - )); - assert_eq!(storage.dump_lease_count(first_id).unwrap(), Some(0)); - assert_eq!(storage.dump_lease_count(next_id).unwrap(), Some(0)); + storage + .conn + .execute( + "UPDATE batches SET invalidated_at_ms=1 WHERE batch_index=0", + [], + ) + .unwrap(); assert_eq!( - storage.gc_unreferenced_dumps().unwrap(), - vec![DumpRow { - id: first_id, - prefix: prefix(0) - }] + storage.latest_snapshot().unwrap().unwrap().dump.prefix, + prefix(0) ); + assert!(storage.gc_unreferenced_dumps().unwrap().is_empty()); + drop(lease); + assert_eq!(storage.gc_unreferenced_dumps().unwrap()[0].id, id); } #[test] - fn failed_snapshot_history_query_does_not_lease_or_arm_a_release() { - let db = temp_db("snapshot-lease-history-query-failure"); - let mut storage = Storage::open(db.path.as_str()).expect("open"); - let finalized_id = storage.insert_finalized_dump(&prefix(0), 100, 0).unwrap(); - let pending_id = storage.insert_pending_dump(&prefix(1), 0, 1).unwrap(); - storage - .conn - .execute_batch("DROP TABLE history_state") + fn failed_lease_commit_never_arms_a_release() { + let db = temp_db("failed-lease-commit"); + let mut storage = Storage::open(&db.path).unwrap(); + let id = storage + .insert_baseline_snapshot(&prefix(0), ExecutedInputCount::ZERO) .unwrap(); - let scheduled = Arc::new(AtomicUsize::new(0)); - + storage.conn.execute_batch( + "CREATE TABLE lease_parent(id INTEGER PRIMARY KEY); + CREATE TABLE lease_child(parent_id INTEGER REFERENCES lease_parent(id) DEFERRABLE INITIALLY DEFERRED); + CREATE TRIGGER fail_lease AFTER UPDATE OF lease_count ON dumps WHEN NEW.lease_count > OLD.lease_count + BEGIN INSERT INTO lease_child VALUES(1); END;" + ).unwrap(); + let count = Arc::new(AtomicUsize::new(0)); + let counter = count.clone(); + let schedule: ReleaseScheduler = Arc::new(move |_| { + counter.fetch_add(1, Ordering::SeqCst); + }); assert!( storage - .acquire_finalized_lease(counting_scheduler(scheduled.clone()), noop_reporter()) + .acquire_latest_snapshot_lease(schedule.clone(), reporter()) .is_err() ); assert!( storage - .acquire_latest_snapshot_lease( - counting_scheduler(scheduled.clone()), - noop_reporter() - ) + .acquire_finalized_lease(schedule, reporter()) .is_err() ); - assert_eq!(scheduled.load(Ordering::SeqCst), 0); - assert_eq!(storage.dump_lease_count(finalized_id).unwrap(), Some(0)); - assert_eq!(storage.dump_lease_count(pending_id).unwrap(), Some(0)); - } - - #[test] - fn acquire_finalized_lease_reads_and_leases_atomically_blocking_gc() { - let db = temp_db("acquire-finalized-lease"); - let mut storage = Storage::open(db.path.as_str()).expect("open"); - - let id_a = storage.insert_finalized_dump(&prefix(0), 100, 5).unwrap(); - let leased = storage - .acquire_finalized_lease(Arc::new(inline), noop_reporter()) - .unwrap() - .expect("a finalized snapshot exists"); - let FinalizedLease { - inclusion_block, - dump: leased, - } = leased; - assert_eq!(leased.prefix, prefix(0)); - assert_eq!(inclusion_block, 100); - assert_eq!(leased.l2_tx_index, 5); - assert_eq!( - storage.dump_lease_count(id_a).unwrap(), - Some(1), - "acquire leased the dump" - ); - - // Supersede A with a promotion so A becomes unreferenced — but the - // held lease must keep GC off it. - storage.insert_pending_dump(&prefix(1), 0, 7).unwrap(); - storage.promote_finalized(0, 101).unwrap(); - let eligible: HashSet = storage - .gc_dump_rows() - .unwrap() - .into_iter() - .map(|row| row.id) - .collect(); - assert!( - !eligible.contains(&id_a), - "lease must block GC of the superseded-but-leased dump" - ); - - // Dropping the leased dump releases the lease via its guard (inline - // here), making the superseded dump GC-eligible. - drop(leased); - assert_eq!( - storage.dump_lease_count(id_a).unwrap(), - Some(0), - "dropping the guard released the lease" - ); - let eligible: HashSet = storage - .gc_dump_rows() - .unwrap() - .into_iter() - .map(|row| row.id) - .collect(); - assert!(eligible.contains(&id_a), "released dump is GC-eligible"); + assert_eq!(count.load(Ordering::SeqCst), 0); + assert_eq!(storage.dump_lease_count(id).unwrap(), Some(0)); } #[test] - fn failed_finalized_lease_commit_never_arms_a_release() { - let db = temp_db("acquire-finalized-commit-failure"); - let mut storage = Storage::open(db.path.as_str()).expect("open"); - let dump_id = storage.insert_finalized_dump(&prefix(0), 100, 5).unwrap(); - install_deferred_lease_commit_failure(db.path.as_str()); - let scheduled = Arc::new(AtomicUsize::new(0)); - - let err = match storage - .acquire_finalized_lease(counting_scheduler(scheduled.clone()), noop_reporter()) - { - Ok(_) => panic!("deferred foreign-key violation must fail COMMIT"), - Err(err) => err, - }; - + fn failed_history_query_never_leases_or_arms_a_release() { + let db = temp_db("failed-lease-query"); + let mut storage = Storage::open(&db.path).unwrap(); + let id = storage + .insert_baseline_snapshot(&prefix(0), ExecutedInputCount::ZERO) + .unwrap(); + storage + .conn + .execute_batch("DROP TABLE history_state") + .unwrap(); + let count = Arc::new(AtomicUsize::new(0)); + let counter = count.clone(); + let schedule: ReleaseScheduler = Arc::new(move |_| { + counter.fetch_add(1, Ordering::SeqCst); + }); assert!( - err.to_string().contains("FOREIGN KEY"), - "expected deferred constraint failure, got: {err}" - ); - assert_eq!( - scheduled.load(Ordering::SeqCst), - 0, - "a rolled-back increment has no matching release to schedule" - ); - assert_eq!( - storage.dump_lease_count(dump_id).unwrap(), - Some(0), - "the failed transaction rolled back the lease increment" + storage + .acquire_latest_snapshot_lease(schedule.clone(), reporter()) + .is_err() ); - } - - #[test] - fn acquire_latest_snapshot_lease_prefers_pending_and_leases() { - let db = temp_db("acquire-latest-pending"); - let mut storage = Storage::open(db.path.as_str()).expect("open"); - - storage.insert_finalized_dump(&prefix(0), 100, 5).unwrap(); - let id_pending = storage.insert_pending_dump(&prefix(1), 3, 9).unwrap(); - - let leased = storage - .acquire_latest_snapshot_lease(Arc::new(inline), noop_reporter()) - .unwrap() - .expect("a snapshot exists"); - assert_eq!(leased.prefix, prefix(1), "prefers the latest pending"); - assert_eq!(leased.l2_tx_index, 9); - - // Clear pending so the leased dump is unreferenced; the held lease - // still blocks GC. - storage.clear_pending_dumps().unwrap(); - let eligible: HashSet = storage - .gc_dump_rows() - .unwrap() - .into_iter() - .map(|row| row.id) - .collect(); - assert!(!eligible.contains(&id_pending), "lease blocks GC"); - - drop(leased); - let eligible: HashSet = storage - .gc_dump_rows() - .unwrap() - .into_iter() - .map(|row| row.id) - .collect(); assert!( - eligible.contains(&id_pending), - "released dump is GC-eligible" + storage + .acquire_finalized_lease(schedule, reporter()) + .is_err() ); + assert_eq!(count.load(Ordering::SeqCst), 0); + assert_eq!(storage.dump_lease_count(id).unwrap(), Some(0)); } #[test] - fn acquire_latest_snapshot_lease_falls_back_to_finalized() { - let db = temp_db("acquire-latest-finalized"); - let mut storage = Storage::open(db.path.as_str()).expect("open"); - - storage.insert_finalized_dump(&prefix(0), 100, 5).unwrap(); - let leased = storage - .acquire_latest_snapshot_lease(Arc::new(inline), noop_reporter()) - .unwrap() - .expect("falls back to finalized"); - assert_eq!(leased.prefix, prefix(0)); - assert_eq!(leased.l2_tx_index, 5); - } - - #[test] - fn failed_latest_snapshot_lease_commit_never_arms_a_release() { - let db = temp_db("acquire-latest-commit-failure"); - let mut storage = Storage::open(db.path.as_str()).expect("open"); - let dump_id = storage.insert_pending_dump(&prefix(0), 3, 9).unwrap(); - install_deferred_lease_commit_failure(db.path.as_str()); - let scheduled = Arc::new(AtomicUsize::new(0)); - - let err = match storage - .acquire_latest_snapshot_lease(counting_scheduler(scheduled.clone()), noop_reporter()) - { - Ok(_) => panic!("deferred foreign-key violation must fail COMMIT"), - Err(err) => err, - }; - - assert!( - err.to_string().contains("FOREIGN KEY"), - "expected deferred constraint failure, got: {err}" - ); - assert_eq!( - scheduled.load(Ordering::SeqCst), - 0, - "a rolled-back increment has no matching release to schedule" - ); - assert_eq!( - storage.dump_lease_count(dump_id).unwrap(), - Some(0), - "the failed transaction rolled back the lease increment" - ); + fn persistent_release_failure_reaches_reporter() { + let db = temp_db("persistent-lease"); + let _storage = Storage::open(&db.path).unwrap(); + let reported = Arc::new(AtomicBool::new(false)); + let flag = reported.clone(); + drop(LeaseGuard { + path: db.path, + dump_id: i64::MAX, + schedule: Arc::new(inline), + report_persistent_failure: Arc::new(move |_| { + flag.store(true, Ordering::SeqCst); + }), + }); + assert!(reported.load(Ordering::SeqCst)); } } diff --git a/sequencer/src/storage/test_helpers.rs b/sequencer/src/storage/test_helpers.rs index 69a5830e..b6bf8fbe 100644 --- a/sequencer/src/storage/test_helpers.rs +++ b/sequencer/src/storage/test_helpers.rs @@ -152,11 +152,22 @@ pub(crate) fn seed_closed_batches(storage: &mut Storage, count: u64) { /// Pull every valid sequenced L2 tx out of storage, dropping the offset. /// Test-only convenience around `ordered_l2_txs_page_from`. pub(crate) fn all_ordered_l2_txs(storage: &mut Storage) -> Vec { + let bounds = storage.history_bounds().expect("history bounds"); storage - .ordered_l2_txs_page_from(0, 1_000_000) - .expect("load all ordered l2 txs") + .canonical_history_page( + sequencer_core::history::HistoryClaim { + version: bounds.version, + next_input: bounds.available_from, + }, + 1_000_000, + ) + .expect("all application inputs") + .rows .into_iter() - .map(|row| row.tx) + .map(|row| match row.context { + super::egress::L2TxContext::UserOp { tx, .. } => SequencedL2Tx::UserOp(tx), + super::egress::L2TxContext::DirectInput { tx, .. } => SequencedL2Tx::Direct(tx), + }) .collect() } diff --git a/tests/benchmarks/README.md b/tests/benchmarks/README.md index bfdebc39..8dcaff37 100644 --- a/tests/benchmarks/README.md +++ b/tests/benchmarks/README.md @@ -79,7 +79,7 @@ cargo run -p benchmarks --bin compare_latest --release -- --results-dir tests/be - Self-contained variants therefore require Foundry's `anvil` binary to be installed locally. - `--max-fee` must be at or above the placeholder app's base fee, or every tx is rejected (`422 EXECUTION_REJECTED`) and the run reports no accepted txs. The error message includes the rejection breakdown and the first rejection body, which names the base fee. - `round_trip_latency` drains existing WS backlog before timing so stale history does not pollute the measurement window. -- `sweep --round-trip` carries `from_offset` forward across rounds to avoid re-reading old WS history. +- `sweep --round-trip` bootstraps each latency observation run from the latest snapshot history claim and drains the remaining backlog before measurement. - If sweep hits `Too many open files`, increase the shell limit (`ulimit -n 4096`) or use a smaller concurrency list. - Self-contained variants automatically build a temp DB, spawn `anvil`, start the sequencer, and persist logs/results under `tests/benchmarks/results`. - For non-self-contained runs, start a sequencer instance first and make sure the benchmark domain matches the sequencer domain. diff --git a/tests/benchmarks/src/bin/round_trip_latency.rs b/tests/benchmarks/src/bin/round_trip_latency.rs index f7b515cd..d27c4095 100644 --- a/tests/benchmarks/src/bin/round_trip_latency.rs +++ b/tests/benchmarks/src/bin/round_trip_latency.rs @@ -46,8 +46,6 @@ struct Args { accounts_file: Option, #[arg(long, default_value_t = DEFAULT_WORKLOAD_TRANSFER_AMOUNT)] transfer_amount: u64, - #[arg(long, default_value_t = 0_u64)] - from_offset: u64, #[arg(long, default_value_t = 45_u64)] duration_secs: u64, /// Number of concurrent workers (one wallet per worker). @@ -113,12 +111,11 @@ async fn main() -> BenchResult<()> { .unwrap_or_else(|| args.endpoint.clone()); println!( - "round-trip config: endpoint={}, self_contained={}, domain_chain_id={}, domain_verifying_contract={}, from_offset={}, duration={}s, concurrency={}, max_fee={}, workload={}", + "round-trip config: endpoint={}, self_contained={}, domain_chain_id={}, domain_verifying_contract={}, duration={}s, concurrency={}, max_fee={}, workload={}", endpoint, args.self_contained, domain.chain_id, domain.verifying_contract, - args.from_offset, args.duration_secs, effective_concurrency, args.max_fee, @@ -132,7 +129,6 @@ async fn main() -> BenchResult<()> { ) }); - let mut ws_from_offset = args.from_offset; let mut nonce_offsets = vec![0_u64; effective_concurrency]; if args.warmup_secs > 0 { @@ -143,7 +139,6 @@ async fn main() -> BenchResult<()> { let warmup_config = RoundTripRunConfig { endpoint: endpoint.clone(), domain, - from_offset: ws_from_offset, duration: Duration::from_secs(args.warmup_secs), concurrency: effective_concurrency, nonce_offsets: nonce_offsets.clone(), @@ -157,7 +152,6 @@ async fn main() -> BenchResult<()> { for (i, advance) in warmup_report.nonce_advances.iter().enumerate() { nonce_offsets[i] += advance; } - ws_from_offset = ws_from_offset.saturating_add(warmup_report.consumed_ws_events_total); println!( "warmup complete: accepted={}, rejected={}", warmup_report.accepted, warmup_report.rejected @@ -167,7 +161,6 @@ async fn main() -> BenchResult<()> { let config = RoundTripRunConfig { endpoint, domain, - from_offset: ws_from_offset, duration: Duration::from_secs(args.duration_secs), concurrency: effective_concurrency, nonce_offsets, @@ -220,7 +213,6 @@ async fn main() -> BenchResult<()> { "domain_version": DOMAIN_VERSION, "domain_chain_id": domain.chain_id, "domain_verifying_contract": domain.verifying_contract.to_string(), - "from_offset": args.from_offset, "duration_secs": args.duration_secs, "warmup_secs": args.warmup_secs, "concurrency": effective_concurrency, diff --git a/tests/benchmarks/src/bin/sweep.rs b/tests/benchmarks/src/bin/sweep.rs index b2d1de03..eed6988a 100644 --- a/tests/benchmarks/src/bin/sweep.rs +++ b/tests/benchmarks/src/bin/sweep.rs @@ -79,9 +79,6 @@ struct Args { /// Max wait time (ms) for remaining WS events after workers finish (round-trip mode only). #[arg(long, default_value_t = 5_000_u64)] max_ws_wait_ms: u64, - /// Initial WS subscribe offset (round-trip mode only). - #[arg(long, default_value_t = 0_u64)] - from_offset: u64, /// Run a warmup phase before the first sweep step. #[arg(long, default_value_t = 0_u64)] warmup_secs: u64, @@ -183,7 +180,6 @@ async fn main() -> BenchResult<()> { let max_workers = concurrencies.iter().copied().max().unwrap_or(1); let mut nonce_offsets = vec![0_u64; max_workers]; - let mut ws_from_offset = args.from_offset; let mut total_accepted = 0_u64; // Warmup phase. @@ -200,7 +196,6 @@ async fn main() -> BenchResult<()> { let warmup_config = RoundTripRunConfig { endpoint: endpoint.clone(), domain, - from_offset: ws_from_offset, duration: warmup_duration, concurrency: first_concurrency, nonce_offsets: offsets_for_warmup, @@ -213,7 +208,6 @@ async fn main() -> BenchResult<()> { for (i, advance) in warmup_report.nonce_advances.iter().enumerate() { nonce_offsets[i] += advance; } - ws_from_offset = ws_from_offset.saturating_add(warmup_report.consumed_ws_events_total); println!( "warmup complete: accepted={}, rejected={}", warmup_report.accepted, warmup_report.rejected @@ -254,7 +248,6 @@ async fn main() -> BenchResult<()> { let config = RoundTripRunConfig { endpoint: endpoint.clone(), domain, - from_offset: ws_from_offset, duration, concurrency, nonce_offsets: offsets_for_step, @@ -268,7 +261,6 @@ async fn main() -> BenchResult<()> { for (i, advance) in report.nonce_advances.iter().enumerate() { nonce_offsets[i] += advance; } - ws_from_offset = ws_from_offset.saturating_add(report.consumed_ws_events_total); print_round_trip_report(&report); RtSweepRow::new( diff --git a/tests/benchmarks/src/round_trip.rs b/tests/benchmarks/src/round_trip.rs index bada7c25..44348587 100644 --- a/tests/benchmarks/src/round_trip.rs +++ b/tests/benchmarks/src/round_trip.rs @@ -30,7 +30,6 @@ const DEFAULT_BACKLOG_DRAIN_MAX_MS: u64 = 2_000; pub struct RoundTripRunConfig { pub endpoint: String, pub domain: BenchmarkDomain, - pub from_offset: u64, pub duration: Duration, pub concurrency: usize, /// Per-worker nonce offsets (worker i starts at nonce_offsets[i]). @@ -84,7 +83,9 @@ pub async fn run_round_trip_benchmark( SequencerClient::new_with_timeout(config.endpoint.clone(), timeout).map_err(|e| { crate::support::io_err(format!("invalid endpoint '{}': {e}", config.endpoint)) })?; - let ws_subscribe_url = client.ws_subscribe_url(config.from_offset); + // This observer tracks latency only; it holds no application state to restore. + let claim = client.latest_snapshot().await?.claim; + let ws_subscribe_url = client.ws_subscribe_url(claim); let domain = config.domain.eip712_domain(); let workers = config.concurrency; @@ -99,7 +100,7 @@ pub async fn run_round_trip_benchmark( )?; // Connect WS and drain backlog. - let mut ws = client.subscribe(config.from_offset).await.map_err(|err| { + let mut ws = client.subscribe(claim).await.map_err(|err| { io_err(format!( "ws connect failed: url={ws_subscribe_url}, error={err}" )) diff --git a/tests/e2e/src/test_cases.rs b/tests/e2e/src/test_cases.rs index 18349ae2..d20de024 100644 --- a/tests/e2e/src/test_cases.rs +++ b/tests/e2e/src/test_cases.rs @@ -291,21 +291,16 @@ pub fn test_cases() -> Vec<(&'static str, ScenarioFn)> { Box::pin(run_stalled_safe_head_live_exit_test(runtime)) }), ( - "ws_reconnect_at_invalidated_offset_skips_cleanly_test", + "ws_reconnect_with_invalidated_claim_is_refused_test", |runtime| { - Box::pin(run_ws_reconnect_at_invalidated_offset_skips_cleanly_test( - runtime, - )) - }, - ), - ( - "ws_subscribe_from_future_offset_waits_silently_test", - |runtime| { - Box::pin(run_ws_subscribe_from_future_offset_waits_silently_test( + Box::pin(run_ws_reconnect_with_invalidated_claim_is_refused_test( runtime, )) }, ), + ("ws_subscribe_ahead_of_head_is_refused_test", |runtime| { + Box::pin(run_ws_subscribe_ahead_of_head_is_refused_test(runtime)) + }), ( "recovery_drains_safe_but_undrained_direct_input_test", |runtime| { @@ -448,8 +443,9 @@ async fn mine_until_finalized_advances( for _ in 0..PROMOTION_POLL_ATTEMPTS { runtime.mine_live_l1_blocks(1).await?; tokio::time::sleep(PROMOTION_POLL_INTERVAL).await; - let (inclusion_block, _) = runtime.finalized_snapshot_info()?; - if inclusion_block > floor { + if let Some(inclusion_block) = runtime.finalized_inclusion_block().await? + && inclusion_block > floor + { return Ok(inclusion_block); } } @@ -563,7 +559,7 @@ async fn drive_finalized_gold_batch_for_watchdog( alice_l2: &mut WalletL2Client, alice_address: Address, ) -> ScenarioResult<()> { - let (floor_inclusion_block, _) = runtime.finalized_snapshot_info()?; + let floor_inclusion_block = runtime.finalized_inclusion_block().await?.unwrap_or(0); let batches_before = runtime.count_batches()?; for _ in 0..TRANSFERS_TO_FORCE_BATCH_CLOSE { alice_l2.transfer(alice_address, U256::from(1_u64)).await?; @@ -765,15 +761,16 @@ async fn run_reconnect_from_offset_test(runtime: &mut ManagedSequencer) -> Scena deposit_amount, ) .await?; - // WS replay is cursor-based and exclusive: `from_offset` means - // "start after this already-consumed DB offset". - let reconnect_offset = deposit_message.offset(); + // The resume claim names the next input, after the consumed deposit. + let reconnect_claim = + runtime.history_claim(deposit_message.offset().checked_add(1).unwrap())?; drop(ws); alice_l2.transfer(bob_address, transfer_amount).await?; bob_l2.withdraw(withdrawal_amount).await?; - let mut resumed_ws = runtime.ws(reconnect_offset).await?; + let mut resumed_ws = + WsClient::connect(&SequencerClient::new(runtime.endpoint())?, reconnect_claim).await?; replay.apply(resumed_ws.expect_user_op_from(alice_address).await?)?; replay.apply(resumed_ws.expect_user_op_from(bob_address).await?)?; @@ -1364,7 +1361,7 @@ async fn drive_promotion_and_capture( runtime: &ManagedSequencer, ) -> ScenarioResult { mine_until_finalized_advances(runtime, 0).await?; - runtime.capture_finalized_checkpoint() + runtime.capture_finalized_checkpoint().await } async fn run_setup_recovery_round_trip_test(runtime: &mut ManagedSequencer) -> ScenarioResult<()> { @@ -1459,7 +1456,7 @@ async fn run_setup_recovery_round_trip_test(runtime: &mut ManagedSequencer) -> S // S' and the resumed submission both land exactly on the canonical chain // state. The local `replay` can't check this (the wiped history is never // re-fed), so the watchdog's independent CM is the authority. - let (floor_inclusion_block, _) = runtime.finalized_snapshot_info()?; + let floor_inclusion_block = runtime.finalized_inclusion_block().await?.unwrap_or(0); let batches_before = runtime.count_batches()?; for _ in 0..TRANSFERS_TO_FORCE_BATCH_CLOSE { alice_l2_after @@ -3276,18 +3273,9 @@ async fn run_stalled_safe_head_live_exit_test( // after the WS connection dropped. // // A WS connection cannot span invalidation: the sequencer necessarily exits -// (danger detection or stop) before any cascade runs (`recover_post_flush` or -// `recover_aging_tip`), and the socket dies with the process. The -// meaningful invariant is the **reconnect** behavior — -// a client that reconnects at `from_offset=N`, where `N` was an offset it -// previously received and whose row is *now invalidated*, must see the -// cursor skip cleanly past `N` and deliver only post-recovery events. -// -// covers the adjacent case (`from_offset=0`), which trivially walks -// `valid_sequenced_l2_txs` from the start. This case is distinct because -// the query `WHERE offset > N` is pointed at an offset that no longer -// exists in the valid view. -async fn run_ws_reconnect_at_invalidated_offset_skips_cleanly_test( +// (danger detection or stop) before any cascade runs. Recovery invalidates +// the identity attached to a previously consumed prefix. +async fn run_ws_reconnect_with_invalidated_claim_is_refused_test( runtime: &mut ManagedSequencer, ) -> ScenarioResult<()> { // Past-stale: matches `recovery_after_stale_batches_test` sizing. @@ -3317,7 +3305,7 @@ async fn run_ws_reconnect_at_invalidated_offset_skips_cleanly_test( .transfer(bob_address, U256::from(100_000_u64)) .await?; let transfer_msg = ws.expect_user_op_from(alice_address).await?; - let last_seen_offset = transfer_msg.offset(); + let resume_claim = runtime.history_claim(transfer_msg.offset().checked_add(1).unwrap())?; replay_before.apply(transfer_msg)?; // Kill the WS socket and the sequencer (same way a real reconnect arc @@ -3328,104 +3316,36 @@ async fn run_ws_reconnect_at_invalidated_offset_skips_cleanly_test( runtime.advance_wall_and_mine(PAST_STALE).await?; runtime.respawn().await?; - // Reconnect at the last offset the client observed — now invalidated. - // The query `WHERE offset > last_seen_offset` against - // `valid_sequenced_l2_txs` must skip cleanly past the invalidated - // rows and deliver only the post-recovery events (the re-drained - // deposit). - let mut ws_after = runtime.ws(last_seen_offset).await?; - let redrained = ws_after - .expect_direct_input_from(runtime.erc20_portal_address()) - .await?; - // The re-drained deposit's offset is strictly greater than the - // last-seen offset — if the cursor ever delivered an invalidated row - // or the same offset again, that'd be the regression. - assert!( - redrained.offset() > last_seen_offset, - "re-drained event must have a strictly-greater offset: \ - last_seen={last_seen_offset}, redrained={}", - redrained.offset(), - ); - ws_after.expect_no_message_for(NO_WS_MESSAGE_WAIT).await?; - - // Sanity check: also reconnecting at 0 produces the same single event - // ('s property), to rule out any one-off weirdness in the - // non-zero reconnect path. - drop(ws_after); - let mut ws_from_zero = runtime.ws(0).await?; - let redrained_from_zero = ws_from_zero + assert!(matches!( + SequencerClient::new(runtime.endpoint())? + .subscribe(resume_claim) + .await, + Err(sequencer_rust_client::SubscribeError::History( + sequencer_rust_client::HistoryPolicyError::StaleGeneration { .. } + )) + )); + let mut replay = runtime.ws(0).await?; + let redrained = replay .expect_direct_input_from(runtime.erc20_portal_address()) .await?; - assert_eq!( - redrained.offset(), - redrained_from_zero.offset(), - "reconnect-at-invalidated and reconnect-at-zero must deliver the \ - same next valid event", - ); - ws_from_zero - .expect_no_message_for(NO_WS_MESSAGE_WAIT) - .await?; + assert_eq!(redrained.offset(), 0); + replay.expect_no_message_for(NO_WS_MESSAGE_WAIT).await?; Ok(()) } -// `from_offset=future` waits silently without erroring. -// -// A subscribe at a far-future offset is a valid subscription that should -// behave the same way `from_offset=0` does on an empty feed: sit idle on -// the live broadcast channel until an event with a greater offset arrives, -// no error, no close. -// -// The behavior is deliberately consistent with `from_offset=0` on an empty -// head — otherwise we'd be making the wait-for-something-new path differ -// based on whether history exists. Test pins this as part of the WS -// subscription contract. -async fn run_ws_subscribe_from_future_offset_waits_silently_test( +async fn run_ws_subscribe_ahead_of_head_is_refused_test( runtime: &mut ManagedSequencer, ) -> ScenarioResult<()> { - // Comfortably beyond any offset this test will produce. `sequenced_l2_txs` - // is rowid-based; rowid_u64 ≤ a few by the end of the short workload. - const FUTURE_OFFSET: u64 = 1_000_000; - // Enough real time to observe "waits silently" without being slow. - const WAIT_WINDOW: Duration = Duration::from_secs(2); - - let alice = TestSigner::from_default(1)?; - let bob = TestSigner::from_default(2)?; - let alice_address = alice.address(); - let bob_address = bob.address(); - - // Seed some actual events so we're not testing "empty head, future - // offset" (trivial case). We want "non-trivial head, offset beyond it". - let alice_l1 = runtime.wallet_l1(alice.clone()).await?; - let mut alice_l2 = runtime.wallet_l2(alice)?; - let mut replay = ReplayWalletApp::devnet(); - { - let mut ws = runtime.ws(0).await?; - apply_reconciled_supported_deposit( - runtime, - &mut ws, - &mut replay, - &alice_l1, - U256::from(500_000_u64), - ) - .await?; - alice_l2.transfer(bob_address, U256::from(1_u64)).await?; - replay.apply(ws.expect_user_op_from(alice_address).await?)?; - } - - // Subscribe far beyond the current head. The subscribe itself must - // succeed (no 4xx / WS close code), and the resulting stream must be - // quiet until something with a greater offset arrives. - let mut ws_future = runtime.ws(FUTURE_OFFSET).await?; - ws_future.expect_no_message_for(WAIT_WINDOW).await?; - - // Generate more activity. These events are still at offsets far below - // `FUTURE_OFFSET`, so they must not be delivered — the subscription - // keeps waiting. - alice_l2.transfer(bob_address, U256::from(1_u64)).await?; - alice_l2.transfer(bob_address, U256::from(1_u64)).await?; - ws_future.expect_no_message_for(WAIT_WINDOW).await?; - + let claim = runtime.history_claim(u64::MAX)?; + assert!(matches!( + SequencerClient::new(runtime.endpoint())? + .subscribe(claim) + .await, + Err(sequencer_rust_client::SubscribeError::History( + sequencer_rust_client::HistoryPolicyError::AheadOfHead { .. } + )) + )); Ok(()) } diff --git a/tests/harness/Cargo.toml b/tests/harness/Cargo.toml index c9cd4146..28befde2 100644 --- a/tests/harness/Cargo.toml +++ b/tests/harness/Cargo.toml @@ -18,6 +18,9 @@ cartesi-rollups-contracts = { workspace = true } futures-util = { workspace = true } k256 = { workspace = true } rusqlite = { workspace = true } +reqwest = { workspace = true, features = ["json"] } +tar = "0.4" +toml = "0.8" sequencer-core = { path = "../../sequencer-core" } sequencer-rust-client = { path = "../../sdk/rust-client" } serde = { workspace = true } diff --git a/tests/harness/src/replay.rs b/tests/harness/src/replay.rs index 723a31bd..ed6a00a5 100644 --- a/tests/harness/src/replay.rs +++ b/tests/harness/src/replay.rs @@ -55,6 +55,14 @@ pub(crate) fn apply_ws_message( app: &mut A, message: WsTxMessage, ) -> HarnessResult<()> { + let expected = app.executed_input_count().get(); + if message.offset() != expected { + return Err(std::io::Error::other(format!( + "WS history gap: expected input {expected}, got {}", + message.offset() + )) + .into()); + } match message { WsTxMessage::DirectInput { sender, diff --git a/tests/harness/src/sequencer.rs b/tests/harness/src/sequencer.rs index 4500d8a5..0ab10434 100644 --- a/tests/harness/src/sequencer.rs +++ b/tests/harness/src/sequencer.rs @@ -397,27 +397,26 @@ impl ManagedSequencer { self.max_batch_open_seconds = secs; } - /// Read the finalized snapshot's `(inclusion_block B, resume_nonce N)`. `B` - /// is `0` for the genesis snapshot and `> 0` once a real batch has been - /// promoted — poll this (mining L1 in between) to wait for a recoverable - /// checkpoint. Read-only. - pub fn finalized_snapshot_info(&self) -> HarnessResult<(u64, u64)> { - let db_path = self.data_dir_path.join("sequencer.db"); - let conn = rusqlite::Connection::open_with_flags( - db_path.as_path(), - rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY, - ) - .map_err(|err| io_other(format!("open DB read-only: {err}")))?; - let (prefix, inclusion_block): (String, i64) = conn - .query_row( - "SELECT d.prefix, f.inclusion_block FROM finalized_snapshot f \ - JOIN dumps d ON d.id = f.dump_id WHERE f.singleton_id = 0", - [], - |row| Ok((row.get(0)?, row.get(1)?)), - ) - .map_err(|err| io_other(format!("read finalized_snapshot: {err}")))?; - let resume_nonce = read_info_next_batch_nonce(Path::new(&prefix))?; - Ok((inclusion_block as u64, resume_nonce)) + /// The currently exportable canonical checkpoint's L1 block. A rebuilt + /// optimistic baseline returns `None` until a matching batch is accepted. + pub async fn finalized_inclusion_block(&self) -> HarnessResult> { + let response = reqwest::Client::new() + .get(format!( + "{}/finalized_state/inclusion_block", + self.endpoint() + )) + .timeout(Duration::from_secs(5)) + .send() + .await?; + if response.status() == reqwest::StatusCode::NOT_FOUND { + return Ok(None); + } + #[derive(serde::Deserialize)] + struct Metadata { + inclusion_block: u64, + } + let metadata: Metadata = response.error_for_status()?.json().await?; + Ok(Some(metadata.inclusion_block)) } /// Read the first frame's persisted fee. This observes startup directly, @@ -475,37 +474,46 @@ impl ManagedSequencer { Ok((frame_safe_block, persisted_safe_head)) } - /// Copy the current finalized snapshot dump to `/checkpoint` (which - /// survives [`Self::reset_database`], since that only clears `sequencer.db*` - /// and `dumps/`), returning the captured checkpoint. Call after a batch has - /// been promoted (`finalized_snapshot_info().0 > 0`). - pub fn capture_finalized_checkpoint(&self) -> HarnessResult { - let db_path = self.data_dir_path.join("sequencer.db"); - let conn = rusqlite::Connection::open_with_flags( - db_path.as_path(), - rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY, - ) - .map_err(|err| io_other(format!("open DB read-only: {err}")))?; - let (prefix, inclusion_block): (String, i64) = conn - .query_row( - "SELECT d.prefix, f.inclusion_block FROM finalized_snapshot f \ - JOIN dumps d ON d.id = f.dump_id WHERE f.singleton_id = 0", - [], - |row| Ok((row.get(0)?, row.get(1)?)), - ) - .map_err(|err| io_other(format!("read finalized_snapshot: {err}")))?; - let src = PathBuf::from(prefix); + /// Download the leased accepted-checkpoint export, including its coherent + /// `checkpoint.toml` receipt, outside the paths cleared by reset_database. + pub async fn capture_finalized_checkpoint(&self) -> HarnessResult { + let response = reqwest::Client::new() + .get(format!("{}/finalized_snapshot", self.endpoint())) + .timeout(Duration::from_secs(30)) + .send() + .await? + .error_for_status()?; + let inclusion_block: u64 = response + .headers() + .get("X-Inclusion-Block") + .ok_or_else(|| io_other("checkpoint export missing X-Inclusion-Block"))? + .to_str()? + .parse()?; + let bytes = response.bytes().await?; let dst = self.data_dir_path.join("checkpoint"); if dst.exists() { - fs::remove_dir_all(dst.as_path()) - .map_err(|err| io_other(format!("clear prior checkpoint ({dst:?}): {err}")))?; + fs::remove_dir_all(&dst)?; } - copy_dir_recursive(src.as_path(), dst.as_path()) - .map_err(|err| io_other(format!("copy checkpoint {src:?} -> {dst:?}: {err}")))?; - let resume_nonce = read_info_next_batch_nonce(dst.as_path())?; + fs::create_dir_all(&dst)?; + tar::Archive::new(bytes.as_ref()).unpack(&dst)?; + #[derive(serde::Deserialize)] + struct Receipt { + inclusion_block: u64, + next_batch_nonce: u64, + } + let receipt: Receipt = toml::from_str(&fs::read_to_string(dst.join("checkpoint.toml"))?)?; + let resume_nonce = read_info_next_batch_nonce(&dst)?; + assert_eq!( + receipt.inclusion_block, inclusion_block, + "checkpoint receipt/header boundary mismatch" + ); + assert_eq!( + receipt.next_batch_nonce, resume_nonce, + "checkpoint receipt/artifact nonce mismatch" + ); Ok(RecoveryCheckpoint { dir: dst, - checkpoint_block: inclusion_block as u64, + checkpoint_block: inclusion_block, resume_nonce, }) } @@ -1112,9 +1120,34 @@ impl ManagedSequencer { std::fs::read_to_string(&self.log_path).map_err(Into::into) } - pub async fn ws(&self, from_offset: u64) -> HarnessResult { + /// Test-only fresh replay claim. Reconnection tests retain their original claim explicitly. + pub fn history_claim( + &self, + next_input: u64, + ) -> HarnessResult { + let conn = rusqlite::Connection::open_with_flags( + self.data_dir_path.join("sequencer.db"), + rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY, + )?; + let (era, generation): (Vec, i64) = conn.query_row( + "SELECT era_id, recovery_generation FROM history_state", + [], + |r| Ok((r.get(0)?, r.get(1)?)), + )?; + Ok(sequencer_rust_client::HistoryClaim { + version: sequencer_rust_client::HistoryVersion { + era_id: sequencer_core::history::EraId::try_from(era.as_slice())?, + recovery_generation: sequencer_core::history::RecoveryGeneration::new( + u64::try_from(generation)?, + ), + }, + next_input: sequencer_rust_client::ExecutedInputCount::new(next_input), + }) + } + + pub async fn ws(&self, next_input: u64) -> HarnessResult { let client = self.sequencer_client()?; - WsClient::connect(&client, from_offset).await + WsClient::connect(&client, self.history_claim(next_input)?).await } pub async fn wallet_l1(&self, signer: TestSigner) -> HarnessResult { @@ -1201,23 +1234,6 @@ fn read_info_next_batch_nonce(dump_dir: &Path) -> HarnessResult { Err(io_other(format!("info.toml missing next_batch_nonce: {info_path:?}")).into()) } -/// Recursively copy a directory tree (used to stash a checkpoint dump where -/// [`ManagedSequencer::reset_database`] won't wipe it). -fn copy_dir_recursive(src: &Path, dst: &Path) -> io::Result<()> { - fs::create_dir_all(dst)?; - for entry in fs::read_dir(src)? { - let entry = entry?; - let from = entry.path(); - let to = dst.join(entry.file_name()); - if entry.file_type()?.is_dir() { - copy_dir_recursive(&from, &to)?; - } else { - fs::copy(&from, &to)?; - } - } - Ok(()) -} - #[allow(clippy::too_many_arguments)] async fn spawn_sequencer_process( sequencer_bin: &Path, diff --git a/tests/harness/src/ws.rs b/tests/harness/src/ws.rs index 97a3033e..53d676e6 100644 --- a/tests/harness/src/ws.rs +++ b/tests/harness/src/ws.rs @@ -6,7 +6,7 @@ use std::time::Duration; use alloy_primitives::Address; use futures_util::StreamExt; use sequencer_core::api::WsTxMessage; -use sequencer_rust_client::SequencerClient; +use sequencer_rust_client::{HistoryClaim, SequencerClient}; use tokio_tungstenite::tungstenite::Message; use crate::HarnessResult; @@ -20,11 +20,10 @@ pub struct WsClient { } impl WsClient { - pub async fn connect(client: &SequencerClient, from_offset: u64) -> HarnessResult { - let stream = - tokio::time::timeout(DEFAULT_WS_CONNECT_TIMEOUT, client.subscribe(from_offset)) - .await - .map_err(|_| io_other("timeout connecting websocket"))??; + pub async fn connect(client: &SequencerClient, claim: HistoryClaim) -> HarnessResult { + let stream = tokio::time::timeout(DEFAULT_WS_CONNECT_TIMEOUT, client.subscribe(claim)) + .await + .map_err(|_| io_other("timeout connecting websocket"))??; Ok(Self { stream }) } diff --git a/watchdog/sequencer_reader.lua b/watchdog/sequencer_reader.lua index b03f96cf..263863ce 100644 --- a/watchdog/sequencer_reader.lua +++ b/watchdog/sequencer_reader.lua @@ -50,8 +50,8 @@ function sequencer_reader.new(http, json, base_url) if type(decoded.inclusion_block) ~= "number" then return nil, "inclusion_block must be a number" end - if type(decoded.l2_tx_index) ~= "number" then - return nil, "l2_tx_index must be a number" + if type(decoded.executed_input_count) ~= "number" then + return nil, "executed_input_count must be a number" end return decoded end @@ -76,8 +76,8 @@ function sequencer_reader.new(http, json, base_url) if not inclusion_block then return nil, inclusion_err end - local l2_tx_index, index_err = parse_header_number(response.headers, "X-L2-Tx-Index") - if not l2_tx_index then + local executed_input_count, index_err = parse_header_number(response.headers, "X-Executed-Input-Count") + if not executed_input_count then return nil, index_err end @@ -88,7 +88,7 @@ function sequencer_reader.new(http, json, base_url) return { inclusion_block = inclusion_block, - l2_tx_index = l2_tx_index, + executed_input_count = executed_input_count, state = response.body, etag = response.headers and response.headers["etag"], } diff --git a/watchdog/tests/drill_divergence.lua b/watchdog/tests/drill_divergence.lua index da3171fa..1be78626 100644 --- a/watchdog/tests/drill_divergence.lua +++ b/watchdog/tests/drill_divergence.lua @@ -64,12 +64,12 @@ local deps = { }, sequencer = { get_finalized_inclusion_block = function() - return { inclusion_block = 1, l2_tx_index = 0 } + return { inclusion_block = 1, executed_input_count = 0 } end, get_finalized_state = function() return { inclusion_block = 1, - l2_tx_index = 0, + executed_input_count = 0, state = string.char(0x01, 0x02, 0x03, 0x04), } end, diff --git a/watchdog/tests/run.lua b/watchdog/tests/run.lua index db27e5fa..d6efbd2f 100644 --- a/watchdog/tests/run.lua +++ b/watchdog/tests/run.lua @@ -701,7 +701,7 @@ test("main tick writes status.prom through exit path", function() }, sequencer = { get_finalized_inclusion_block = function() - return { inclusion_block = 5, l2_tx_index = 0 } + return { inclusion_block = 5, executed_input_count = 0 } end, }, machine = fake_machine("{}"), @@ -827,7 +827,7 @@ test("successful idle compare writes ok status prom", function() }, sequencer = { get_finalized_inclusion_block = function() - return { inclusion_block = 5, l2_tx_index = 0 } + return { inclusion_block = 5, executed_input_count = 0 } end, }, machine = fake_machine("{}"), @@ -886,12 +886,12 @@ test("tick writes status.prom into watchdog state dir", function() }, sequencer = { get_finalized_inclusion_block = function() - return { inclusion_block = 2, l2_tx_index = 0 } + return { inclusion_block = 2, executed_input_count = 0 } end, get_finalized_state = function() return { inclusion_block = 2, - l2_tx_index = 0, + executed_input_count = 0, state = '{"a":1}', } end, @@ -1074,12 +1074,12 @@ test("runner happy path replays inputs and writes checkpoint", function() checkpoint = checkpoint_mod, sequencer = { get_finalized_inclusion_block = function() - return { inclusion_block = 12, l2_tx_index = 0 } + return { inclusion_block = 12, executed_input_count = 0 } end, get_finalized_state = function() return { inclusion_block = 12, - l2_tx_index = 0, + executed_input_count = 0, state = '{"ok":true}', } end, @@ -1123,12 +1123,12 @@ test("runner advances CM as streamed input chunks arrive", function() checkpoint = checkpoint_mod, sequencer = { get_finalized_inclusion_block = function() - return { inclusion_block = 12, l2_tx_index = 0 } + return { inclusion_block = 12, executed_input_count = 0 } end, get_finalized_state = function() return { inclusion_block = 12, - l2_tx_index = 0, + executed_input_count = 0, state = '{"ok":true}', } end, @@ -1181,12 +1181,12 @@ test("runner advances CM over empty streamed partitions", function() }, sequencer = { get_finalized_inclusion_block = function() - return { inclusion_block = 12, l2_tx_index = 0 } + return { inclusion_block = 12, executed_input_count = 0 } end, get_finalized_state = function() return { inclusion_block = 12, - l2_tx_index = 0, + executed_input_count = 0, state = '{"ok":true}', } end, @@ -1226,12 +1226,12 @@ test("runner returns state mismatch payload", function() }, sequencer = { get_finalized_inclusion_block = function() - return { inclusion_block = 2, l2_tx_index = 0 } + return { inclusion_block = 2, executed_input_count = 0 } end, get_finalized_state = function() return { inclusion_block = 2, - l2_tx_index = 0, + executed_input_count = 0, state = '{"a":1}', } end, @@ -1379,7 +1379,7 @@ test("runner returns transient error when L1 RPC head lags target block", functi }, sequencer = { get_finalized_inclusion_block = function() - return { inclusion_block = 10, l2_tx_index = 0 } + return { inclusion_block = 10, executed_input_count = 0 } end, }, rpc = { @@ -1408,12 +1408,12 @@ test("runner returns transient error when finalized inclusion_block moves during }, sequencer = { get_finalized_inclusion_block = function() - return { inclusion_block = 1, l2_tx_index = 0 } + return { inclusion_block = 1, executed_input_count = 0 } end, get_finalized_state = function() return { inclusion_block = 2, - l2_tx_index = 0, + executed_input_count = 0, state = string.char(1), } end, @@ -1441,7 +1441,7 @@ test("runner skips compare cycle when finalized inclusion_block is unchanged", f }, sequencer = { get_finalized_inclusion_block = function() - return { inclusion_block = 5, l2_tx_index = 0 } + return { inclusion_block = 5, executed_input_count = 0 } end, get_finalized_state = function() error("get_finalized_state must not run when inclusion_block is unchanged") @@ -1469,12 +1469,12 @@ test("runner returns sequencer inclusion_block regression payload", function() }, sequencer = { get_finalized_inclusion_block = function() - return { inclusion_block = 4, l2_tx_index = 0 } + return { inclusion_block = 4, executed_input_count = 0 } end, get_finalized_state = function() return { inclusion_block = 4, - l2_tx_index = 0, + executed_input_count = 0, state = "{}", } end, @@ -1493,7 +1493,7 @@ test("sequencer client reads finalized inclusion_block", function() assert_eq(url, "http://sequencer/finalized_state/inclusion_block") return { status = 200, - body = '{"inclusion_block":7,"l2_tx_index":3}', + body = '{"inclusion_block":7,"executed_input_count":3}', headers = {}, } end @@ -1501,7 +1501,7 @@ test("sequencer client reads finalized inclusion_block", function() function json.decode(body) return { inclusion_block = 7, - l2_tx_index = 3, + executed_input_count = 3, } end @@ -1509,7 +1509,7 @@ test("sequencer client reads finalized inclusion_block", function() local head, err = client:get_finalized_inclusion_block() assert(head, err) assert_eq(head.inclusion_block, 7) - assert_eq(head.l2_tx_index, 3) + assert_eq(head.executed_input_count, 3) end) test("sequencer client reads finalized SSZ body and headers", function() @@ -1521,7 +1521,7 @@ test("sequencer client reads finalized SSZ body and headers", function() body = "raw-state", headers = { ["x-inclusion-block"] = "9", - ["x-l2-tx-index"] = "1", + ["x-executed-input-count"] = "1", }, } end @@ -1534,7 +1534,7 @@ test("sequencer client reads finalized SSZ body and headers", function() local state, err = client:get_finalized_state() assert(state, err) assert_eq(state.inclusion_block, 9) - assert_eq(state.l2_tx_index, 1) + assert_eq(state.executed_input_count, 1) assert_eq(state.state, "raw-state") end) From 7f3229f2e42585e055f2fabb8e280c4d42dd5a81 Mon Sep 17 00:00:00 2001 From: gcdepaula Date: Wed, 16 Sep 2026 19:53:21 -0300 Subject: [PATCH 10/29] test: validate Track 3 cold replica and canonical recovery Exercise nonempty HTTP restore, concurrent catch-up, live replication, and recovery rebootstrap through the consumer API. Preserve Lua module paths in doctor and correct benchmark fee defaults uncovered during validation. Record pinned emulator 0.20 canonical-machine gates and an exact-revision ABBA latency comparison. Native engine conformance and deployment latency remain separate integration gates. --- Cargo.lock | 1 + docs/plans/2026-07-coordination-tracks.md | 16 +- .../2026-07-track3-feed-replay-design.md | 12 +- docs/review/2026-09-16-track3-validation.md | 152 ++++++++++ docs/review/register.md | 1 + tests/benchmarks/README.md | 19 +- tests/benchmarks/justfile | 12 +- .../benchmarks/src/bin/round_trip_latency.rs | 4 +- tests/benchmarks/src/bin/sweep.rs | 2 +- tests/e2e/Cargo.toml | 1 + tests/e2e/src/cold_replica.rs | 282 ++++++++++++++++++ tests/e2e/src/lib.rs | 1 + tests/e2e/src/test_cases.rs | 6 +- tests/harness/src/replay.rs | 14 +- watchdog/justfile | 2 +- 15 files changed, 493 insertions(+), 32 deletions(-) create mode 100644 docs/review/2026-09-16-track3-validation.md create mode 100644 tests/e2e/src/cold_replica.rs diff --git a/Cargo.lock b/Cargo.lock index 81d393d9..d0ce0bdd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3882,6 +3882,7 @@ dependencies = [ "sequencer-core", "sequencer-rust-client", "serde_json", + "tar", "tempfile", "tokio", "tracing-subscriber", diff --git a/docs/plans/2026-07-coordination-tracks.md b/docs/plans/2026-07-coordination-tracks.md index 7509931c..85222678 100644 --- a/docs/plans/2026-07-coordination-tracks.md +++ b/docs/plans/2026-07-coordination-tracks.md @@ -23,10 +23,9 @@ freely at this stage — no backward-compatibility constraints. **Current campaign order:** -1. Review and validate the integrated application-history, snapshot, and Track 3 cutover. -2. Validate Track 6 against the reference C bridge, then the private DEX engine when shared. -3. Run recovery/watchdog end-to-end gates and remeasure feed latency in the representative environment. -4. Track 5 (fee LUT) only after the log-space-fees decision. +1. Validate Track 6 against the reference C bridge, then the private DEX engine when shared. +2. Exercise native-engine snapshot bootstrap and remeasure feed latency in the representative environment. +3. Track 5 (fee LUT) only after the log-space-fees decision. Full restore archives now support file and directory application prefixes. Additional snapshot retention or transport mechanisms require a measured consumer need. @@ -47,10 +46,11 @@ available backlog is replayable without a total catch-up cap, with bounded pages queues, and subscribers. Recovery refuses old claims before delivering inputs. The former physical replay cursor and sparse attribution design are superseded -by the [application-history design](application-history.md). Native-engine -bootstrap, representative latency measurements, and environment-dependent -recovery/watchdog runs remain integration gates; no additional protocol layer -is assumed for them. +by the [application-history design](application-history.md). The wallet's cold +replica and canonical recovery/watchdog gates have a +[validation record](../review/2026-09-16-track3-validation.md). Native-engine +bootstrap and representative latency measurements remain integration gates; +no additional protocol layer is assumed for them. ## Track 5 — Fee exponentiation LUT (deferred) diff --git a/docs/plans/2026-07-track3-feed-replay-design.md b/docs/plans/2026-07-track3-feed-replay-design.md index 6da85a27..f4546f49 100644 --- a/docs/plans/2026-07-track3-feed-replay-design.md +++ b/docs/plans/2026-07-track3-feed-replay-design.md @@ -114,14 +114,20 @@ Snapshot integration tests own artifact/header association and restore proof. The Anvil recovery/WS gate also exercises process restart, generation refusal, and re-drained direct replay at a reused offset. +The cold-replica E2E restores a nonempty HTTP archive, checks it against a +genesis-fed replica, and holds catch-up behind a barrier while new writes commit. +It checks whole-state, count, and clock agreement through live direct inputs +and user operations, then exercises real stale recovery, claim refusal, and +fresh bootstrap. Canonical-machine gates cover genesis, ordinary execution, +stale recovery, and database reconstruction from an exported checkpoint. +The [validation record](../review/2026-09-16-track3-validation.md) records the +pinned environment and local latency measurements. + Remaining integration gates are concrete consumers and environments: - Validate the native reference adapter and, when available, the private DEX bridge against the application contract and this bootstrap workflow. - Remeasure submit-to-matching-WS-event latency on the representative deployment. -- Complete the broader recovery/watchdog scenarios in the pinned emulator - environment. The native Anvil recovery gate does not prove canonical-machine - comparison; the local host currently has emulator 0.21 while the repo pins 0.20. Revisit resumable snapshot transfer only when artifact size requires it; retained client checkpoints only when full rebootstrap cost matters; archival diff --git a/docs/review/2026-09-16-track3-validation.md b/docs/review/2026-09-16-track3-validation.md new file mode 100644 index 00000000..d44cdb78 --- /dev/null +++ b/docs/review/2026-09-16-track3-validation.md @@ -0,0 +1,152 @@ +# Track 3 integration validation — 2026-09-16 + +Scope: validate application-history commit +`799a5d3aba71d7ec1c5f48fda3b05bfd536b1a11` with the pinned canonical machine, +a complete cold replica, and a same-host latency comparison against +`91e25780854bb641c63135751f951f9f7ee1e744`. + +## Environment and canonical agreement + +The shared development flake was switched to emulator 0.20.0 using its previously +recorded source, generated-files, and uarch hashes. The CLI, Lua module, and +native library all resolved to the same Nix package. Existing unrelated Foundry +edits were preserved; the shared flake lock was unchanged. This environment edit +lives outside the sequencer repository. + +Rust 1.95.0, Lua 5.4.7, and Foundry 1.5.1 were used. A fresh devnet canonical +image was built from this checkout with the pinned cross image and kernel; +no 0.21 machine archive was reused. + +| Artifact | Hash | +|---|---| +| Canonical machine root | `fb93d09eeb69b2fbd1fc63232d428e285fa0a2000430c256e9e7489e0b89df96` | +| Devnet guest binary SHA-256 | `154250d5095bfb507314cc3291c8679a43427223c6642d3440ad6e35142e74d4` | +| Root filesystem SHA-256 | `e92485f9b34ba7cbbab9e750b55d2ffa32dd18afabecbc6ddef6108fcd275ee5` | +| Pinned Anvil fixture SHA-256 | `b140e31db2b04bb99c733fdf153718cd252335370f4b355849e2cbb3121fc30f` | + +All four selected canonical-machine gates passed: + +| Scenario | What it checks | Time | +|---|---|---:| +| `watchdog_genesis_compare_test` | Native genesis bytes equal CM inspection; production watchdog initializes and idles twice | 1.94 s | +| `deposit_transfer_withdrawal_test` | Ordinary application execution and non-genesis watchdog comparison | 10.49 s | +| `recovery_after_stale_batches_test` | Stale-batch recovery followed by independent from-genesis CM comparison | 11.38 s | +| `setup_recovery_round_trip_test` | Real `/finalized_snapshot` download, database wipe, `setup --recovery`, resumed execution, and independent CM comparison | 15.36 s | + +These are selected integration gates, not a claim that the entire E2E suite or +private DEX adapter was tested. The existing 693-test host-suite result belongs +to the implementation record in the register. + +## Cold replica + +Added `cold_replica_snapshot_backlog_live_recovery_test` and two small wallet +replay helpers. The scenario passed in 18.12 seconds, including its test-owned +120-second deadline. Its claims come from HTTP headers and consumed inputs, +without querying storage for the consumer's history identity. + +The test restores a nonempty tar archive, deletes the downloaded source, and +compares all application bytes/count/clock with an independently accumulated +genesis replay. Writes commit after snapshot selection but before restoration, +and between restoration and subscription. A two-way barrier pauses the consumer +after its first backlog entry until more writes have committed, proving that +producer progress overlaps incomplete client catch-up. It then verifies live +direct inputs and user operations. + +An actual stale outage/restart invalidates the suffix. The old claim receives +`STALE_GENERATION`; a new archive has the same era and the next generation. +The expected replacement branch retains the accepted prefix and replays only +its retained L1 directs. Optimistic transfers disappear, and a new transfer at +the recovered nonce succeeds. + +## Tooling fixes + +- `just doctor` preserves Lua's configured search paths, matching the production + watchdog. Its forced Linux-only environment variables hid the Nix Cartesi + module. The corrected doctor loads both lcurl and the new machine image. +- Benchmark CLI/recipe defaults use `max_fee=2000`. The former 1200 default was + below the self-contained frame fee of 1356, rejecting every request. The stale + `--from-offset` help example was also corrected. An initial benchmark attempt + with 1200 was discarded during warmup; both compared revisions use an explicit + 2000 limit. + +## Latency comparison + +Four release-build runs used an ABBA order: baseline, current, current, +baseline. Each had 5 seconds of warmup and a 45-second measured window, +16 closed-loop workers, funded transfers of one unit, one WS observer, an +explicit max fee of 2000, a 3-second request deadline, and a 5-second WS deadline. +The host was an Apple M5 Max (18 logical CPUs, 36 GiB) running macOS 26.6.2. +No builds, correctness tests, or injected network shaping ran during measurement. + +Both exact revisions used their matching SDK/protocol and the same fresh machine +image, Anvil fixture, and toolchain. Per-request latency excludes funding, +startup, snapshot acquisition, backlog draining, and signing. RSS was sampled +with `ps` every 500 ms over warmup and measured traffic. These are local regression +measurements, not a fixed-arrival capacity test or a deployment/network SLO. +The harness correctly marked its network-aware target as `not_evaluated`. + +| Run order | Revision | Accepted and WS-matched | TPS | Peak RSS (MiB) | +|---|---|---:|---:|---:| +| 1. baseline-1 | `91e2578` | 45,846 | 1017.46 | 23.88 | +| 2. current-1 | `799a5d3` | 45,696 | 1014.04 | 23.64 | +| 3. current-2 | `799a5d3` | 45,562 | 1010.98 | 23.36 | +| 4. baseline-2 | `91e2578` | 45,368 | 1006.40 | 23.66 | + +| Run | Metric (ms) | p50 | p95 | p99 | p99.9 | +|---|---|---:|---:|---:|---:| +| baseline-1 | ACK | 14.938 | 27.268 | 28.724 | 40.252 | +| baseline-1 | Matching WS | 28.807 | 44.223 | 53.596 | 59.817 | +| current-1 | ACK | 14.975 | 27.209 | 28.139 | 37.738 | +| current-1 | Matching WS | 29.246 | 44.255 | 53.123 | 57.818 | +| current-2 | ACK | 14.990 | 27.232 | 28.123 | 34.405 | +| current-2 | Matching WS | 29.147 | 44.413 | 53.827 | 57.566 | +| baseline-2 | ACK | 15.020 | 27.361 | 28.275 | 39.366 | +| baseline-2 | Matching WS | 29.059 | 44.445 | 53.115 | 57.615 | + +All 182,472 measured requests were accepted, with no client failures and a +matching WS event for each. Per-run latency and throughput overlap: there is no +clear regression at this workload. This does not bound a larger application's +execution/dump cost, deep-history replay, many subscribers, or deployment network +latency. Percentiles above belong to individual runs, not a pooled distribution. + +Build both exact checkouts in release mode, retain each matching binary pair, +and run the following against a fresh self-contained stack in ABBA order: + +```sh +cargo build --release --locked -p wallet-sequencer --bin wallet-sequencer-devnet -p benchmarks --bin round_trip_latency +target/release/round_trip_latency --self-contained \ + --sequencer-bin "$PWD/target/release/wallet-sequencer-devnet" \ + --accounts-file tests/benchmarks/anvil_1000_accounts.txt \ + --duration-secs 45 --warmup-secs 5 --concurrency 16 --max-fee 2000 \ + --request-timeout-ms 3000 --max-ws-wait-ms 5000 \ + --evaluate --json-out RUN.json +``` + +## Checks + +- Workspace/all-target check and strict workspace/all-target/all-feature Clippy. +- Workspace formatting and diff whitespace checks. +- Watchdog unit tests: 62/62. +- Fresh image build, `just doctor`, and the five integration scenarios above. +- A short benchmark smoke run with the corrected implicit fee default: + 138 accepted, zero rejected, and 138 matching WS events. + +In the local shared environment, commands use +`direnv exec /Users/gcdepaula/projects/cartesi-dev/sequencer` before the command: + +```sh +just setup +just canonical-build-machine-image +just doctor +just test-watchdog +cargo build -p wallet-sequencer --bin wallet-sequencer-devnet -p rollups-e2e --bin rollups-e2e --locked +# Repeat for each scenario in the tables above. +target/debug/rollups-e2e SCENARIO --exact --nocapture +cargo check --workspace --all-targets --locked +cargo clippy --workspace --all-targets --all-features --locked -- -D warnings +cargo fmt --all --check +``` + +Native reference-bridge/DEX conformance and representative deployment latency +remain separate integration work. Sparse snapshots, archival replay, and +resumable transfers remain deferred until a consumer requires them. diff --git a/docs/review/register.md b/docs/review/register.md index dc318a90..fadd3892 100644 --- a/docs/review/register.md +++ b/docs/review/register.md @@ -706,3 +706,4 @@ for `2026-06-10-correctness-review.md`, `2026-06-10-simplification.md`, | 2026-09-09 | Application, inclusion lane, and public DEX integration branch | Native progress ownership, typed validation failures, mutable independent checkpoints, optional canonical inspection, and lane bookkeeping simplified; ingress CORS and Lua 5.4 parity restored. Reference C bridge port kept separate. | [Application/lane review](2026-09-09-application-lane-dex-review.md); current Application and snapshot contracts. Workspace check, strict Clippy, 697 host tests, and 62 watchdog tests passed; private DEX conformance remains unverified. | | 2026-09-11 | Reference C bridge port and review boundary | Keep the current Application contract, runtime payload bound, paired progress, filesystem-owned checkpoint disposal, and host failure fixes together. Engine-dependent API refinements and external-engine conformance remain follow-ups. | Settled decisions and owed tests above; [C binding guide](../protocol/c-application-binding.md), Application contract, and snapshot lifecycle. | | 2026-09-16 | Application-history and Track 3 implementation, with independent storage/recovery/snapshot review | Replaced mixed replay and sparse attribution with current application inputs; complete atomic baselines; acceptance-derived immutable snapshots; HTTP restore/recovery archives and mandatory WS claims. Review narrowed equal-block recovery to empty genesis to avoid losing same-block pending directs. | [History design](../plans/application-history.md), current invariants/API/snapshot/recovery docs. Validation: 693 workspace tests, strict Clippy, formatting, 62 watchdog tests, admission TLC (155 distinct states), and the Anvil recovery/old-claim refusal gate passed. Full canonical watchdog comparison remains blocked by the local emulator 0.21 vs repository 0.20 environment; no pin changed. | +| 2026-09-16 | Track 3 integration validation | Nonempty HTTP cold replica, concurrent backlog/live consumption, stale recovery/rebootstrap, and four real canonical-machine gates pass under emulator 0.20. Tooling fixes preserve Lua paths and make benchmark fee defaults admissible. | [Validation and latency evidence](2026-09-16-track3-validation.md); native bridge/DEX and representative deployment latency remain separate gates. | diff --git a/tests/benchmarks/README.md b/tests/benchmarks/README.md index 8dcaff37..79c3a695 100644 --- a/tests/benchmarks/README.md +++ b/tests/benchmarks/README.md @@ -34,25 +34,26 @@ just --justfile tests/benchmarks/justfile all just --justfile tests/benchmarks/justfile all-and-compare ``` -The `just` recipes default `max_fee=1200`, which is above the placeholder app's -base fee. A run whose `--max-fee` is below the base fee has **every** tx -rejected (`422 EXECUTION_REJECTED: "max fee N below base fee ..."`) and reports -no accepted txs — set a fee at or above the base fee. +The CLI and `just` recipes default `max_fee=2000`, above the self-contained +harness's initial frame fee of 1356. External deployments can charge a different +fee. A run whose `--max-fee` is below the frame fee rejects its transactions +with `422 EXECUTION_REJECTED`; choose a sufficient limit and require accepted +transactions and matching WS events before interpreting latency results. Direct `cargo` examples: ```bash # Round-trip latency, self-contained (spawns anvil + sequencer). Runs are # time-bounded (`--duration-secs`), not count-bounded. -cargo run -p benchmarks --bin round_trip_latency --release -- --self-contained --duration-secs 30 --concurrency 4 --max-fee 1200 -cargo run -p benchmarks --bin round_trip_latency --release -- --self-contained --duration-secs 60 --concurrency 16 --max-fee 1200 --evaluate +cargo run -p benchmarks --bin round_trip_latency --release -- --self-contained --duration-secs 30 --concurrency 4 --max-fee 2000 +cargo run -p benchmarks --bin round_trip_latency --release -- --self-contained --duration-secs 60 --concurrency 16 --max-fee 2000 --evaluate # Against an external sequencer (pass that deployment's EIP-712 domain): -cargo run -p benchmarks --bin round_trip_latency --release -- --endpoint http://127.0.0.1:3000 --domain-chain-id 31337 --domain-verifying-contract 0x1111111111111111111111111111111111111111 --duration-secs 30 --concurrency 4 --max-fee 1200 +cargo run -p benchmarks --bin round_trip_latency --release -- --endpoint http://127.0.0.1:3000 --domain-chain-id 31337 --domain-verifying-contract 0x1111111111111111111111111111111111111111 --duration-secs 30 --concurrency 4 --max-fee 2000 # Concurrency sweep — ack latency by default, round-trip with `--round-trip`: -cargo run -p benchmarks --bin sweep --release -- --self-contained --duration-secs 30 --max-fee 1200 --concurrency-list "1 2 4 8 16 32 64 128" -cargo run -p benchmarks --bin sweep --release -- --round-trip --self-contained --duration-secs 30 --max-fee 1200 --concurrency-list "1 2 4 8" +cargo run -p benchmarks --bin sweep --release -- --self-contained --duration-secs 30 --max-fee 2000 --concurrency-list "1 2 4 8 16 32 64 128" +cargo run -p benchmarks --bin sweep --release -- --round-trip --self-contained --duration-secs 30 --max-fee 2000 --concurrency-list "1 2 4 8" # Aggregate the JSON artifacts / compare the two latest of a kind: cargo run -p benchmarks --bin report --release -- --results-dir tests/benchmarks/results diff --git a/tests/benchmarks/justfile b/tests/benchmarks/justfile index 834b800a..492290b6 100644 --- a/tests/benchmarks/justfile +++ b/tests/benchmarks/justfile @@ -16,26 +16,26 @@ clean: ensure-machine-image: test -d {{template_machine_image}} || { echo "missing {{template_machine_image}}; run 'just canonical-build-machine-image' first"; exit 1; } -bench-ack-self duration="45" max_fee="1200" extra="": ensure-machine-image +bench-ack-self duration="45" max_fee="2000" extra="": ensure-machine-image cargo build -p wallet-sequencer --release cargo run -p benchmarks --bin sweep --release -- --self-contained --duration-secs {{duration}} --max-fee {{max_fee}} --concurrency-list 1 --warmup-secs 5 --accounts-file {{accounts_file}} {{extra}} -bench-round-trip-self duration="45" max_fee="1200" concurrency="1" extra="": ensure-machine-image +bench-round-trip-self duration="45" max_fee="2000" concurrency="1" extra="": ensure-machine-image cargo build -p wallet-sequencer --release cargo run -p benchmarks --bin round_trip_latency --release -- --self-contained --duration-secs {{duration}} --max-fee {{max_fee}} --concurrency {{concurrency}} {{extra}} -bench-sweep domain_chain_id verifying_contract duration="45" url="http://127.0.0.1:3000" max_fee="1200" conc_list="1 2 4 8 16 32 64 128 256" extra="": +bench-sweep domain_chain_id verifying_contract duration="45" url="http://127.0.0.1:3000" max_fee="2000" conc_list="1 2 4 8 16 32 64 128 256" extra="": cargo run -p benchmarks --bin sweep --release -- --endpoint {{url}} --domain-chain-id {{domain_chain_id}} --domain-verifying-contract {{verifying_contract}} --duration-secs {{duration}} --max-fee {{max_fee}} --concurrency-list "{{conc_list}}" {{extra}} -bench-sweep-self duration="45" max_fee="1200" conc_list="1 2 4 8 16 32 64 128 256" extra="": ensure-machine-image +bench-sweep-self duration="45" max_fee="2000" conc_list="1 2 4 8 16 32 64 128 256" extra="": ensure-machine-image cargo build -p wallet-sequencer --release cargo run -p benchmarks --bin sweep --release -- --self-contained --duration-secs {{duration}} --max-fee {{max_fee}} --concurrency-list "{{conc_list}}" --accounts-file {{accounts_file}} {{extra}} -bench-rt-sweep-self duration="45" max_fee="1200" conc_list="1 2 4 8" extra="": ensure-machine-image +bench-rt-sweep-self duration="45" max_fee="2000" conc_list="1 2 4 8" extra="": ensure-machine-image cargo build -p wallet-sequencer --release cargo run -p benchmarks --bin sweep --release -- --round-trip --self-contained --duration-secs {{duration}} --max-fee {{max_fee}} --concurrency-list "{{conc_list}}" --accounts-file {{accounts_file}} {{extra}} -bench-capacity-sweep-self duration="45" max_fee="1200" conc_list="1 2 4 8 16 32 64 128 256 512" out="tests/benchmarks/results/capacity-sweep-self.json" extra="": ensure-machine-image +bench-capacity-sweep-self duration="45" max_fee="2000" conc_list="1 2 4 8 16 32 64 128 256 512" out="tests/benchmarks/results/capacity-sweep-self.json" extra="": ensure-machine-image cargo build -p wallet-sequencer --release cargo run -p benchmarks --bin sweep --release -- --self-contained --duration-secs {{duration}} --max-fee {{max_fee}} --concurrency-list {{conc_list}} --json-out {{out}} --accounts-file {{accounts_file}} {{extra}} diff --git a/tests/benchmarks/src/bin/round_trip_latency.rs b/tests/benchmarks/src/bin/round_trip_latency.rs index d27c4095..830cfb9b 100644 --- a/tests/benchmarks/src/bin/round_trip_latency.rs +++ b/tests/benchmarks/src/bin/round_trip_latency.rs @@ -27,7 +27,7 @@ const FUNDING_ESTIMATE_TXS_PER_WORKER: u64 = 100_000; name = "round_trip_latency", about = "round-trip latency benchmark", version, - after_help = "Examples:\n cargo run -p benchmarks --bin round_trip_latency -- --self-contained --duration-secs 30 --concurrency 16 --max-fee 0 --from-offset 0\n cargo run -p benchmarks --bin round_trip_latency -- --self-contained --duration-secs 60 --concurrency 16 --evaluate" + after_help = "Examples:\n cargo run -p benchmarks --bin round_trip_latency -- --self-contained --duration-secs 30 --concurrency 16 --max-fee 2000\n cargo run -p benchmarks --bin round_trip_latency -- --self-contained --duration-secs 60 --concurrency 16 --evaluate" )] struct Args { #[arg(long, default_value = DEFAULT_ENDPOINT)] @@ -51,7 +51,7 @@ struct Args { /// Number of concurrent workers (one wallet per worker). #[arg(long, default_value_t = 1_usize)] concurrency: usize, - #[arg(long, default_value_t = 1200_u16)] + #[arg(long, default_value_t = 2000_u16)] max_fee: u16, #[arg(long, default_value_t = 3_000_u64)] request_timeout_ms: u64, diff --git a/tests/benchmarks/src/bin/sweep.rs b/tests/benchmarks/src/bin/sweep.rs index eed6988a..57744158 100644 --- a/tests/benchmarks/src/bin/sweep.rs +++ b/tests/benchmarks/src/bin/sweep.rs @@ -54,7 +54,7 @@ struct Args { accounts_file: Option, #[arg(long, default_value_t = DEFAULT_WORKLOAD_TRANSFER_AMOUNT)] transfer_amount: u64, - #[arg(long, default_value_t = 1200_u16)] + #[arg(long, default_value_t = 2000_u16)] max_fee: u16, #[arg( long, diff --git a/tests/e2e/Cargo.toml b/tests/e2e/Cargo.toml index 28ec9424..8b5845ac 100644 --- a/tests/e2e/Cargo.toml +++ b/tests/e2e/Cargo.toml @@ -27,3 +27,4 @@ ssz = { workspace = true } tokio = { workspace = true, features = ["macros", "rt-multi-thread", "time", "net", "process", "io-util", "signal"] } serde_json = { workspace = true } tempfile = { workspace = true } +tar = "0.4" diff --git a/tests/e2e/src/cold_replica.rs b/tests/e2e/src/cold_replica.rs new file mode 100644 index 00000000..364888b9 --- /dev/null +++ b/tests/e2e/src/cold_replica.rs @@ -0,0 +1,282 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +//! A cold consumer restores an HTTP archive and follows its claim through replay, +//! live writes, and a process-boundary recovery. Its reference starts at genesis. + +use std::time::Duration; + +use alloy_primitives::U256; +use rollups_harness::{ManagedSequencer, ReplayWalletApp, TestSigner, WsClient}; +use sequencer_core::api::WsTxMessage; +use sequencer_rust_client::{ + HistoryClaim, HistoryPolicyError, SequencerClient, SnapshotResponse, SubscribeError, +}; + +use crate::ScenarioResult; +use crate::test_cases::advance_live_frame_until_covers; + +pub(crate) async fn run(runtime: &mut ManagedSequencer) -> ScenarioResult<()> { + tokio::time::timeout(Duration::from_secs(120), run_scenario(runtime)) + .await + .map_err(|_| "cold replica scenario exceeded its 120-second deadline")? +} + +async fn run_scenario(runtime: &mut ManagedSequencer) -> ScenarioResult<()> { + let client = SequencerClient::new(runtime.endpoint())?; + let genesis = client.latest_snapshot().await?; + assert_eq!(genesis.claim.next_input.get(), 0); + let mut reference_ws = WsClient::connect(&client, genesis.claim).await?; + drop(genesis); + let mut reference = ReplayWalletApp::devnet(); + let mut history = Vec::new(); + + let alice = TestSigner::from_default(1)?; + let alice_address = alice.address(); + let bob_address = TestSigner::from_default(2)?.address(); + let alice_l1 = runtime.wallet_l1(alice.clone()).await?; + let mut alice_l2 = runtime.wallet_l2(alice)?; + let deposit = U256::from(100_000_000_u64); + let deposit_block = alice_l1.mint_and_deposit_supported_token(deposit).await?; + advance_live_frame_until_covers(runtime, deposit_block).await?; + record(&mut reference_ws, &mut reference, &mut history).await?; + + // Size closure supplies a nonempty immutable snapshot while leaving a suffix + // in the next batch. Only the first batch is submitted before the outage. + for _ in 0..150 { + alice_l2.transfer(alice_address, U256::from(1)).await?; + record(&mut reference_ws, &mut reference, &mut history).await?; + } + wait_for_accepted_snapshot(runtime).await?; + let snapshot = client.latest_snapshot().await?; + let original_claim = snapshot.claim; + let snapshot_count = original_claim.next_input.get(); + assert!( + snapshot_count > 0, + "the consumer must restore nonempty state" + ); + assert!(snapshot_count <= reference.executed_input_count()); + + // Hold the response without consuming its body. This write is acknowledged + // after snapshot selection and before archive restoration or subscription. + alice_l2.transfer(bob_address, U256::from(1_000)).await?; + record(&mut reference_ws, &mut reference, &mut history).await?; + let (mut replica, downloaded_claim) = restore(snapshot).await?; + assert_eq!(downloaded_claim, original_claim); + let snapshot_reference = replay_prefix(&history, snapshot_count)?; + assert_same_state(&replica, &snapshot_reference)?; + assert!(replica.executed_input_count() < reference.executed_input_count()); + + let backlog_deposit = alice_l1 + .mint_and_deposit_supported_token(U256::from(70_000)) + .await?; + advance_live_frame_until_covers(runtime, backlog_deposit).await?; + record(&mut reference_ws, &mut reference, &mut history).await?; + alice_l2.transfer(bob_address, U256::from(2_000)).await?; + record(&mut reference_ws, &mut reference, &mut history).await?; + + let mut replica_ws = WsClient::connect(&client, downloaded_claim).await?; + let backlog_head = reference.executed_input_count(); + let catch_up_target = backlog_head + 2; + let (first_replayed, replay_started) = tokio::sync::oneshot::channel(); + let (writes_committed, resume_replay) = tokio::sync::oneshot::channel(); + let producer = async { + replay_started.await?; + for amount in [3_000_u64, 4_000] { + alice_l2.transfer(bob_address, U256::from(amount)).await?; + record(&mut reference_ws, &mut reference, &mut history).await?; + } + writes_committed + .send(()) + .map_err(|_| "consumer dropped the commit barrier")?; + ScenarioResult::Ok(()) + }; + let consumer = async { + replica.apply(replica_ws.next_message().await?)?; + assert!(replica.executed_input_count() < backlog_head); + first_replayed + .send(()) + .map_err(|_| "producer dropped the replay barrier")?; + resume_replay.await?; + consume_until(&mut replica_ws, &mut replica, catch_up_target).await + }; + futures::try_join!(producer, consumer)?; + assert_same_state(&replica, &reference)?; + replica_ws + .expect_no_message_for(Duration::from_millis(100)) + .await?; + + // Already at the tip: these inputs must arrive through live continuation. + let clock_before_live = replica.last_executed_safe_block(); + let live_deposit = alice_l1 + .mint_and_deposit_supported_token(U256::from(80_000)) + .await?; + advance_live_frame_until_covers(runtime, live_deposit).await?; + record(&mut reference_ws, &mut reference, &mut history).await?; + alice_l2.transfer(bob_address, U256::from(5_000)).await?; + record(&mut reference_ws, &mut reference, &mut history).await?; + consume_until( + &mut replica_ws, + &mut replica, + reference.executed_input_count(), + ) + .await?; + assert_same_state(&replica, &reference)?; + assert!(replica.last_executed_safe_block() > clock_before_live); + assert_eq!( + replica.current_user_balance(bob_address), + U256::from(15_000) + ); + + let stale_claim = HistoryClaim { + next_input: sequencer_rust_client::ExecutedInputCount::new(replica.executed_input_count()), + ..downloaded_claim + }; + drop(replica_ws); + drop(reference_ws); + runtime.stop().await?; + runtime + .advance_wall_and_mine(Duration::from_secs( + (sequencer_core::MAX_WAIT_BLOCKS + 50) * 12, + )) + .await?; + runtime.respawn().await?; + + // Respawn chooses a fresh listener. The claim comes exclusively from the + // downloaded snapshot and consumed inputs, never the harness's DB helpers. + let client = SequencerClient::new(runtime.endpoint())?; + assert!(matches!( + client.subscribe(stale_claim).await, + Err(SubscribeError::History( + HistoryPolicyError::StaleGeneration { .. } + )) + )); + let (mut recovered, fresh_claim) = restore(client.latest_snapshot().await?).await?; + assert_eq!(fresh_claim.version.era_id, original_claim.version.era_id); + assert_eq!( + fresh_claim.version.recovery_generation.get(), + original_claim.version.recovery_generation.get() + 1, + ); + assert_eq!(fresh_claim.next_input.get(), snapshot_count); + + // Reconstruct the expected replacement branch independently: the accepted + // prefix survives, optimistic user ops disappear, and L1 directs replay. + let mut recovered_reference = replay_prefix(&history, snapshot_count)?; + assert_same_state(&recovered, &recovered_reference)?; + for message in &history[snapshot_count as usize..] { + if let WsTxMessage::DirectInput { .. } = message { + let mut direct = message.clone(); + if let WsTxMessage::DirectInput { offset, .. } = &mut direct { + *offset = recovered_reference.executed_input_count(); + } + recovered_reference.apply(direct)?; + } + } + let mut recovered_ws = WsClient::connect(&client, fresh_claim).await?; + consume_until( + &mut recovered_ws, + &mut recovered, + recovered_reference.executed_input_count(), + ) + .await?; + assert_same_state(&recovered, &recovered_reference)?; + assert_eq!(recovered.current_user_balance(bob_address), U256::ZERO); + assert!(recovered.executed_input_count() < replica.executed_input_count()); + + let mut alice_l2 = runtime.wallet_l2(TestSigner::from_default(1)?)?; + alice_l2.set_next_nonce(recovered_reference.current_user_nonce(alice_address)); + alice_l2.transfer(bob_address, U256::from(6_000)).await?; + let resumed = recovered_ws.expect_user_op_from(alice_address).await?; + recovered.apply(resumed.clone())?; + recovered_reference.apply(resumed)?; + assert_same_state(&recovered, &recovered_reference)?; + assert_eq!( + recovered.current_user_balance(bob_address), + U256::from(6_000) + ); + recovered_ws + .expect_no_message_for(Duration::from_millis(100)) + .await?; + Ok(()) +} + +async fn restore(snapshot: SnapshotResponse) -> ScenarioResult<(ReplayWalletApp, HistoryClaim)> { + let claim = snapshot.claim; + assert_eq!( + snapshot.response.headers()["Content-Type"], + "application/x-tar" + ); + let archive = snapshot.response.bytes().await?; + let directory = tempfile::tempdir()?; + tar::Archive::new(archive.as_ref()).unpack(directory.path())?; + assert!(directory.path().join("info.toml").is_file()); + let app = ReplayWalletApp::from_dump(&directory.path().join("state"))?; + assert_eq!(app.executed_input_count(), claim.next_input.get()); + // Subsequent replay also checks that restoring does not retain a dependency + // on the downloaded source directory. + directory.close()?; + Ok((app, claim)) +} + +async fn record( + ws: &mut WsClient, + reference: &mut ReplayWalletApp, + history: &mut Vec, +) -> ScenarioResult<()> { + let message = ws.next_message().await?; + reference.apply(message.clone())?; + history.push(message); + Ok(()) +} + +async fn consume_until( + ws: &mut WsClient, + app: &mut ReplayWalletApp, + target: u64, +) -> ScenarioResult<()> { + while app.executed_input_count() < target { + app.apply(ws.next_message().await?)?; + } + assert_eq!(app.executed_input_count(), target); + Ok(()) +} + +fn replay_prefix(history: &[WsTxMessage], count: u64) -> ScenarioResult { + let mut app = ReplayWalletApp::devnet(); + for message in &history[..usize::try_from(count)?] { + app.apply(message.clone())?; + } + Ok(app) +} + +fn assert_same_state(actual: &ReplayWalletApp, expected: &ReplayWalletApp) -> ScenarioResult<()> { + assert_eq!( + actual.executed_input_count(), + expected.executed_input_count() + ); + assert_eq!( + actual.last_executed_safe_block(), + expected.last_executed_safe_block() + ); + assert_eq!( + actual.canonical_snapshot_bytes()?, + expected.canonical_snapshot_bytes()?, + "all wallet state, including balances, nonces, config, count, and clock" + ); + Ok(()) +} + +async fn wait_for_accepted_snapshot(runtime: &ManagedSequencer) -> ScenarioResult<()> { + for _ in 0..40 { + runtime.mine_live_l1_blocks(1).await?; + if runtime + .finalized_inclusion_block() + .await? + .is_some_and(|b| b > 0) + { + return Ok(()); + } + tokio::time::sleep(Duration::from_secs(1)).await; + } + Err("timed out waiting for the initial batch's accepted snapshot".into()) +} diff --git a/tests/e2e/src/lib.rs b/tests/e2e/src/lib.rs index 5b720177..96ef4a2b 100644 --- a/tests/e2e/src/lib.rs +++ b/tests/e2e/src/lib.rs @@ -1,6 +1,7 @@ // (c) Cartesi and individual authors (see AUTHORS) // SPDX-License-Identifier: Apache-2.0 (see LICENSE) +mod cold_replica; pub mod test_cases; mod watchdog_compare; diff --git a/tests/e2e/src/test_cases.rs b/tests/e2e/src/test_cases.rs index d20de024..e04e86e2 100644 --- a/tests/e2e/src/test_cases.rs +++ b/tests/e2e/src/test_cases.rs @@ -152,6 +152,10 @@ struct ExpectedWalletState { pub fn test_cases() -> Vec<(&'static str, ScenarioFn)> { vec![ + ( + "cold_replica_snapshot_backlog_live_recovery_test", + |runtime| Box::pin(crate::cold_replica::run(runtime)), + ), ("deposit_transfer_withdrawal_test", |runtime| { Box::pin(run_deposit_transfer_withdrawal_test(runtime)) }), @@ -473,7 +477,7 @@ async fn mine_until_batch_is_safe_accepted( /// wait for the input reader and inclusion lane to commit a frame covering /// `required_safe_block`. The live safe-head query prevents stale SQLite /// observations from turning an asynchronous reader poll into over-mining. -async fn advance_live_frame_until_covers( +pub(crate) async fn advance_live_frame_until_covers( runtime: &ManagedSequencer, required_safe_block: u64, ) -> ScenarioResult { diff --git a/tests/harness/src/replay.rs b/tests/harness/src/replay.rs index ed6a00a5..4b1bf678 100644 --- a/tests/harness/src/replay.rs +++ b/tests/harness/src/replay.rs @@ -4,7 +4,9 @@ use alloy_primitives::{Address, U256}; use app_core::application::{WalletApp, WalletConfig}; use sequencer_core::api::WsTxMessage; -use sequencer_core::application::{Application, execute_direct_input, execute_valid_user_op}; +use sequencer_core::application::{ + Application, CanonicalState, execute_direct_input, execute_valid_user_op, +}; use sequencer_core::l2_tx::{DirectInput, ValidUserOp}; use crate::HarnessResult; @@ -14,6 +16,16 @@ pub struct ReplayWalletApp { } impl ReplayWalletApp { + pub fn from_dump(prefix: &std::path::Path) -> HarnessResult { + Ok(Self { + app: WalletApp::from_dump(prefix)?, + }) + } + + pub fn canonical_snapshot_bytes(&self) -> HarnessResult> { + Ok(self.app.canonical_snapshot_bytes()?) + } + pub fn devnet() -> Self { Self { app: WalletApp::new(WalletConfig::devnet()), diff --git a/watchdog/justfile b/watchdog/justfile index 0817c2bd..1d62360d 100644 --- a/watchdog/justfile +++ b/watchdog/justfile @@ -31,5 +31,5 @@ doctor: exit 1; \ } @"${CARTESI_WATCHDOG_LUA_BIN:-lua5.4}" -e "package.cpath = '.deps/lua/?.so;' .. package.cpath; require('lcurl'); print('doctor: lcurl ok')" - @LUA_CPATH_5_4="/usr/lib/lua/5.4/?.so;/usr/lib/lua/5.4/?/init.so;${LUA_CPATH_5_4:-;}" LUA_PATH_5_4="/usr/share/lua/5.4/?.lua;/usr/share/lua/5.4/?/init.lua;${LUA_PATH_5_4:-;}" "${CARTESI_WATCHDOG_LUA_BIN:-lua5.4}" -e "package.path = './?.lua;./?/init.lua;' .. package.path; local m=require('watchdog.machine_cartesi').new(); local inst,err=m:load('examples/canonical-app/out/canonical-machine-image'); if not inst then error('doctor: machine_cartesi cannot load devnet image: '..tostring(err)) end; print('doctor: machine_cartesi load ok')" + @"${CARTESI_WATCHDOG_LUA_BIN:-lua5.4}" -e "package.path = './?.lua;./?/init.lua;' .. package.path; local m=require('watchdog.machine_cartesi').new(); local inst,err=m:load('examples/canonical-app/out/canonical-machine-image'); if not inst then error('doctor: machine_cartesi cannot load devnet image: '..tostring(err)) end; print('doctor: machine_cartesi load ok')" @echo "doctor: watchdog toolchain ok" From 4b6810af7556a215ad658d297d7c4f9cbd82ccdb Mon Sep 17 00:00:00 2001 From: gcdepaula Date: Thu, 17 Sep 2026 10:55:07 -0300 Subject: [PATCH 11/29] refactor: simplify host fee price estimation --- README.md | 2 +- sequencer/src/l1/fee_oracle/uniswap.rs | 178 ++++++------------ .../src/storage/migrations/0001_schema.sql | 2 +- 3 files changed, 57 insertions(+), 125 deletions(-) diff --git a/README.md b/README.md index fe414f9f..3df30f2d 100644 --- a/README.md +++ b/README.md @@ -264,7 +264,7 @@ publication of the accepted checkpoint. See [snapshot lifecycle](docs/snapshots/ - `safe_inputs`: every raw InputBox observation, including batch envelopes - `history_state`: immutable era baseline (application count and accounted L1 block) plus recovery generation - `snapshots` and `dumps`: immutable batch-close/baseline artifacts and streaming leases; accepted status is derived from `safe_accepted_batches` -- `batch_policy`: singleton knobs and constants for DA-style batch sizing and fee derivation; `batch_policy_derived` exposes `recommended_fee` and `batch_size_target`. A batch closes on whichever fires first: the derived `batch_size_target` byte budget or the `max_batch_open` wall-clock deadline (an inclusion-lane setting, `CARTESI_SEQUENCER_MAX_BATCH_OPEN_SECONDS`, not a `batch_policy` column). Setup writes the first `log_gas_price` (and observation stamp) for both Fixed and Uniswap modes, failing if the initial Uniswap quote cannot be read. Fixed local pricing has no oracle worker; Uniswap starts from the persisted price and refreshes lazily via the setup-pinned WETH/fee-token TWAP source, retaining that price across transient source failures. `log_slack = log(10)` applies the 10× safety margin in log space. Fees are app-token smallest units — initially USDC (6 decimals) for the wallet prototype — not a protocol-level USDC invariant. +- `batch_policy`: singleton knobs and constants for DA-style batch sizing and fee derivation; `batch_policy_derived` exposes `recommended_fee` and `batch_size_target`. A batch closes on whichever fires first: the derived `batch_size_target` byte budget or the `max_batch_open` wall-clock deadline (an inclusion-lane setting, `CARTESI_SEQUENCER_MAX_BATCH_OPEN_SECONDS`, not a `batch_policy` column). Setup writes the first `log_gas_price` (and observation stamp) for both Fixed and Uniswap modes, failing if the initial Uniswap quote cannot be read. Fixed local pricing has no oracle worker; Uniswap starts from the persisted price and refreshes lazily via the setup-pinned WETH/fee-token TWAP source, retaining that price across transient source failures. Host tick-to-price conversion uses approximate floating-point arithmetic before checked integer gas-cost calculation and log encoding; recorded frame fees execute with deterministic integer arithmetic. `log_slack = log(10)` applies the 10× safety margin in log space. Fees are app-token smallest units — initially USDC (6 decimals) for the wallet prototype — not a protocol-level USDC invariant. ## Project Layout diff --git a/sequencer/src/l1/fee_oracle/uniswap.rs b/sequencer/src/l1/fee_oracle/uniswap.rs index 04169d20..a67917b3 100644 --- a/sequencer/src/l1/fee_oracle/uniswap.rs +++ b/sequencer/src/l1/fee_oracle/uniswap.rs @@ -6,7 +6,7 @@ use alloy::contract::Error as ContractError; use alloy::providers::{DynProvider, Provider}; use alloy::sol; -use alloy_primitives::{Address, U256, Uint}; +use alloy_primitives::{Address, U256}; use alloy_sol_types::Revert; use async_trait::async_trait; use thiserror::Error; @@ -229,106 +229,26 @@ pub(super) fn bootstrap_price_source_error(error: PriceSourceError) -> (bool, St } } -/// Convert a Uniswap tick to fee-token smallest units per WETH. +/// Estimate fee-token smallest units per WETH from a Uniswap tick. +/// +/// Host pricing is approximate, with headroom supplied by `batch_policy.log_slack`. +/// Ceiling rounds up the estimate, not an exact TickMath price. Recorded frame +/// fees use the shared deterministic integer conversion. pub fn quote_x_per_weth_from_tick( tick: i32, weth_is_token0: bool, ) -> Result { - let sqrt_price_x96 = sqrt_ratio_at_tick(tick)?; - let square: Uint<512, 8> = sqrt_price_x96.widening_mul(sqrt_price_x96); - let q192: Uint<512, 8> = Uint::from(1u8) << 192; - let one_weth = Uint::<512, 8>::from(1_000_000_000_000_000_000u128); - let (numerator, denominator) = if weth_is_token0 { - (square * one_weth, q192) - } else { - (q192 * one_weth, square) - }; - ceil_div_512(numerator, denominator) -} - -fn sqrt_ratio_at_tick(tick: i32) -> Result { const MAX_TICK: i32 = 887_272; - if tick.unsigned_abs() > MAX_TICK as u32 { - return Err(PriceSourceError::ArithmeticOverflow); - } - let constants = [ - "fffcb933bd6fad37aa2d162d1a594001", - "fff97272373d413259a46990580e213a", - "fff2e50f5f656932ef12357cf3c7fdcc", - "ffe5caca7e10e4e61c3624eaa0941cd0", - "ffcb9843d60f6159c9db58835c926644", - "ff973b41fa98c081472e6896dfb254c0", - "ff2ea16466c96a3843ec78b326b52861", - "fe5dee046a99a2a811c461f1969c3053", - "fcbe86c7900a88aedcffc83b479aa3a4", - "f987a7253ac413176f2b074cf7815e54", - "f3392b0822b70005940c7a398e4b70f3", - "e7159475a2c29b7443b29c7fa6e889d9", - "d097f3bdfd2022b8845ad8f792aa5825", - "a9f746462d870fdf8a65dc1f90e061e5", - "70d869a156d2a1b890bb3df62baf32f7", - "31be135f97d08fd981231505542fcfa6", - "9aa508b5b7a84e1c677de54f3e99bc9", - "5d6af8dedb81196699c329225ee604", - "2216e584f5fa1ea926041bedfe98", - "48a170391f7dc42444e8fa2", - ]; - let abs = tick.unsigned_abs(); - let mut ratio = if abs & 1 != 0 { - parse_u256(constants[0])? - } else { - U256::from_limbs([0, 0, 1, 0]) - }; - for (bit, constant) in constants.iter().enumerate().skip(1) { - if abs & (1 << bit) != 0 { - ratio = mul_shift_128(ratio, parse_u256(constant)?)?; - } - } - if tick > 0 { - ratio = U256::MAX / ratio; - } - // Q128.128 to Q64.96, rounding up as canonical TickMath does. - let shifted = ratio >> 32; - Ok(if ratio & U256::from((1u64 << 32) - 1) == U256::ZERO { - shifted - } else { - shifted - .checked_add(U256::from(1)) - .ok_or(PriceSourceError::ArithmeticOverflow)? - }) -} - -fn parse_u256(value: &str) -> Result { - U256::from_str_radix(value, 16).map_err(|_| PriceSourceError::ArithmeticOverflow) -} - -fn mul_shift_128(left: U256, right: U256) -> Result { - let shifted: Uint<512, 8> = left.widening_mul(right) >> 128; - let limbs = shifted.into_limbs(); - if limbs[4..].iter().any(|limb| *limb != 0) { - return Err(PriceSourceError::ArithmeticOverflow); - } - Ok(U256::from_limbs([limbs[0], limbs[1], limbs[2], limbs[3]])) -} - -fn ceil_div_512( - numerator: Uint<512, 8>, - denominator: Uint<512, 8>, -) -> Result { - if denominator == Uint::ZERO { + if !(-MAX_TICK..=MAX_TICK).contains(&tick) { return Err(PriceSourceError::ArithmeticOverflow); } - let quotient = numerator / denominator; - let rounded = if numerator % denominator == Uint::ZERO { - quotient - } else { - quotient + Uint::from(1u8) - }; - let limbs = rounded.into_limbs(); - if limbs[4..].iter().any(|limb| *limb != 0) { + let directed_tick = if weth_is_token0 { tick } else { -tick }; + // The raw ratio already uses token smallest units; 1 WETH supplies 10^18 wei. + let quote = 1.0001_f64.powi(directed_tick) * 1e18; + if !quote.is_finite() || quote <= 0.0 { return Err(PriceSourceError::ArithmeticOverflow); } - Ok(U256::from_limbs([limbs[0], limbs[1], limbs[2], limbs[3]])) + U256::try_from(quote.ceil()).map_err(|_| PriceSourceError::ArithmeticOverflow) } #[cfg(test)] @@ -348,13 +268,10 @@ mod tests { } #[test] - fn negative_tick_uses_uniswap_rounding_direction() { - // The exact ratio is below one, so WETH as token1 (inverse quote) is - // above one WETH unit; this also exercises the negative TickMath path. - assert!( - quote_x_per_weth_from_tick(-1, false).unwrap() - > U256::from(1_000_000_000_000_000_000u128) - ); + fn negative_tick_directionality_matches_token_order() { + let one_weth = U256::from(1_000_000_000_000_000_000u128); + assert!(quote_x_per_weth_from_tick(-1, true).unwrap() < one_weth); + assert!(quote_x_per_weth_from_tick(-1, false).unwrap() > one_weth); } #[test] @@ -366,14 +283,14 @@ mod tests { #[test] fn out_of_range_ticks_overflow() { - assert!(matches!( - quote_x_per_weth_from_tick(887_273, true), - Err(PriceSourceError::ArithmeticOverflow) - )); - assert!(matches!( - quote_x_per_weth_from_tick(-887_273, false), - Err(PriceSourceError::ArithmeticOverflow) - )); + for tick in [-887_273, 887_273, i32::MIN, i32::MAX] { + for weth_is_token0 in [false, true] { + assert!(matches!( + quote_x_per_weth_from_tick(tick, weth_is_token0), + Err(PriceSourceError::ArithmeticOverflow) + )); + } + } } #[test] @@ -454,8 +371,16 @@ mod tests { #[test] fn boundary_ticks_are_representable() { - assert!(quote_x_per_weth_from_tick(887_272, true).is_ok()); - assert!(quote_x_per_weth_from_tick(-887_272, false).is_ok()); + for weth_is_token0 in [false, true] { + let tick = if weth_is_token0 { 887_272 } else { -887_272 }; + assert!( + quote_x_per_weth_from_tick(tick, weth_is_token0).unwrap() > U256::from(u128::MAX) + ); + assert_eq!( + quote_x_per_weth_from_tick(-tick, weth_is_token0).unwrap(), + U256::from(1) + ); + } } #[test] @@ -466,19 +391,26 @@ mod tests { } #[test] - fn tick_math_matches_uniswap_canonical_sqrt_ratios() { - // Canonical TickMath.getSqrtRatioAtTick vectors from Uniswap v3-core. - assert_eq!( - sqrt_ratio_at_tick(0).unwrap(), - U256::from(79_228_162_514_264_337_593_543_950_336u128) - ); - assert_eq!( - sqrt_ratio_at_tick(-887_272).unwrap(), - U256::from(4_295_128_739u64) - ); - assert_eq!( - sqrt_ratio_at_tick(887_272).unwrap(), - U256::from_str_radix("1461446703485210103287273052203988822378723970342", 10).unwrap() - ); + fn quotes_preserve_smallest_units_across_token_orders() { + // Decimal reference values: ~2063 USDC/WETH for a 6-decimal token, + // ~2.718 tokens/WETH for an 18-decimal token, and the largest quote. + for (directed_tick, expected) in [ + (-200_000, 2_063_215_670.0), + (10_000, 2.718_145_926_825_225e18), + (887_272, 3.402_567_868_363_881e56), + ] { + for weth_is_token0 in [false, true] { + let tick = if weth_is_token0 { + directed_tick + } else { + -directed_tick + }; + let quote = f64::from(quote_x_per_weth_from_tick(tick, weth_is_token0).unwrap()); + assert!( + (quote - expected).abs() <= expected * 1e-8, + "tick {tick}, WETH token0 {weth_is_token0}: {quote} vs {expected}" + ); + } + } } } diff --git a/sequencer/src/storage/migrations/0001_schema.sql b/sequencer/src/storage/migrations/0001_schema.sql index e94c8c27..a9650de0 100644 --- a/sequencer/src/storage/migrations/0001_schema.sql +++ b/sequencer/src/storage/migrations/0001_schema.sql @@ -519,7 +519,7 @@ END; -- `gas_price` is denominated in "fee-token smallest units per L1 gas unit" -- (application-defined ERC-20 X; the wallet prototype starts with USDC). -- The L1 fee oracle converts base+priority gas (wei) via a pinned Uniswap V3 --- WETH/X TWAP and encodes the exact quote to log space. The tenfold safety +-- WETH/X TWAP and encodes the approximate quote to log space. The tenfold safety -- margin lives in `log_slack` (not in the oracle). Local Anvil uses an -- explicit fixed exponent instead of Uniswap. -- From 254d2365330846e35fb8bd57879c42d6c25d7c73 Mon Sep 17 00:00:00 2001 From: gcdepaula Date: Thu, 17 Sep 2026 11:11:34 -0300 Subject: [PATCH 12/29] docs: clarify recovery and improve agent reading paths --- AGENTS.md | 347 ++++++++++++++++++------------------ CLAUDE.md | 74 +------- README.md | 50 ++++-- docs/recovery/README.md | 55 +++--- docs/recovery/cockroach.md | 245 +++++++++++++------------ docs/snapshots/README.md | 25 ++- docs/threat-model/README.md | 15 +- 7 files changed, 388 insertions(+), 423 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 22add5bb..720c22d5 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,6 +1,9 @@ # AGENTS.md -This file tells AI coding agents and human contributors how to work effectively in this repository. Start here. +Start here for the repository's mental model and working rules. Read this baseline +once, then use [Reading Routes](#reading-routes) to follow the contracts relevant +to the work. Those documents own detailed behavior; this guide explains why it +matters and where to look. ## Mission @@ -72,7 +75,10 @@ Scheduler-acceptance semantics exist in exactly three implementations that must 2. the off-chain acceptance predicate — `ProtocolTiming::scheduler_accepts` ([`sequencer-core/src/protocol.rs`](sequencer-core/src/protocol.rs)), which feeds `safe_accepted_batches`; 3. the inclusion lane's live prediction (drain + execution order). -The expected-nonce fold is homed next to `scheduler_accepts` as `advance_expected_batch_nonce` (same file); the submitter's `decide_submit_start` consumes it, and `populate_safe_accepted_batches` keeps a deliberate inline copy (its advance is interleaved with storage-only side effects — the content-identity check and the divergence freeze — that can't move below the protocol layer). Touching any of these means re-checking the others — their agreement is the system's most load-bearing invariant (see [`docs/invariants.md`](docs/invariants.md)). +The submitter's expected-nonce scan also depends on this agreement. The +[scheduler contract](docs/protocol/scheduler-semantics.md#the-three-implementations-and-why-they-agree) +maps the implementations and the deliberate storage-local copy. Changing one +requires checking the others. Two mechanical facts the agreement rests on: @@ -88,44 +94,40 @@ A batch is **stale** when `inclusion_block - first_frame.safe_block >= MAX_WAIT_ 1. **Liveness failure** — the sequencer went offline and failed to submit batches in time. 2. **Censorship** — the sequencer kept submitting batches but froze `safe_block` to hold back direct inputs. -When the scheduler encounters a stale batch, it **skips it entirely** — no nonce consumed, no state change. This is the **censorship-resistance backstop**: the sequencer cannot hold write priority indefinitely without advancing the drain cursor. Direct inputs are force-drained at `MAX_WAIT_BLOCKS`, guaranteeing deposit availability within ~4h even under adversarial conditions. +When the scheduler encounters a stale batch, it skips its frames without +consuming the batch nonce. The overdue-direct backstop still runs. Together, +these rules prevent the sequencer from holding write priority indefinitely +without advancing the drain cursor; direct inputs are force-drained at +`MAX_WAIT_BLOCKS`, giving the ~4h censorship-resistance bound. ### Cascading invalidation If a batch is stale, all existing subsequent batches are also invalid. The scheduler's expected-nonce counter does not advance on a stale skip, so every subsequent batch arrives at an unexpected nonce and is rejected. Invalidation is a suffix operation: marking batch `N` invalid cascades to `N+1`, `N+2`, …, including the open batch. New batches created after recovery are unaffected. -### Preemptive recovery - -Rather than waiting for a batch to go stale on L1, the sequencer uses a **danger threshold** (`MAX_WAIT_BLOCKS − MARGIN`). The threshold is *only a trigger*: it tells the system "stop running, hand off to recovery." It does not encode "this batch is doomed" — that decision belongs to the post-flush cascade. - -The cycle crosses a process boundary by design: the in-process -[`DangerDetector`](sequencer/src/recovery/detector.rs) polls -`Storage::check_danger` on a cadence and returns a non-`Safe` worker exit; the -runtime closes intake and drains before the command returns non-zero (stopping -the process is how the sequencer goes offline). Diagnosed terminal runtime -faults abort immediately; expected-recovery and retryable exits are graceful; -the orchestrator respawns; on every boot startup recovery re-derives the -response from local facts. - -The authoritative dispatch table, phase ordering, boot-local witnesses, the -admission sequence, the "everything past gold is doomed" model, and the -per-path rationale live in -[`docs/recovery/README.md`](docs/recovery/README.md) — that document **owns** -the recovery design; this section is only the map. Do not restate dispatch -details here. - -### Detection: safe-only, with wall-clock fallback - -Staleness is only checked against L1 **safe** state, never latest. Stale batches in latest that haven't reached safe yet will eventually become safe, and the check will fire at that point. This avoids reacting to L1 reorgs. - -When the sequencer's view of L1 stops advancing — most often because the RPC gateway is stalled or returning stale reads, occasionally because L1 itself is unhealthy — the DB-based staleness check sees a frozen `current_safe_block` and may fail to trigger. The danger detector uses two wall-clock signals: the recorded L1 safe block timestamp must remain younger than `CARTESI_SEQUENCER_L1_READ_STALE_AFTER_BLOCKS`, and unresolved batches are also checked with `estimated_missed_blocks = (now − last_safe_progress_ms) / seconds_per_block` by adjusting the danger threshold downward. This prevents silently issuing doomed soft confirmations during stale-provider periods or L1 outages. - -### Formal verification - -The recovery design is verified by two bounded TLA+ models; -[`docs/recovery/README.md`](docs/recovery/README.md) "Formal Verification" -says what each proves. When touching recovery code, read both current models -first. +### Two recovery paths + +**Automatic recovery** repairs optimistic history after liveness failures. The +danger detector signals service to stop when a danger check fires; startup recovery +settles outstanding submissions and replaces the invalid suffix using local +SQLite facts and safe L1 history. The danger threshold is a trigger, not proof +that a batch is doomed. Detection uses safe state, with wall-clock checks when +the L1 view stops advancing. Expected recovery exits gracefully; diagnosed +terminal runtime faults abort immediately. The [automatic recovery design](docs/recovery/README.md) +owns detection, startup ordering, dispatch, and admission. + +**Manual cockroach recovery** (`setup --recovery`) rebuilds from a trusted +canonical application checkpoint and L1 history into a fresh data directory. +It also applies after sequencer bugs: fix the bug, choose a trusted canonical +checkpoint, and then run recovery. Historical batches, +including malformed ones, receive the canonical scheduler's treatment. The +result accounts for a fixed input prefix from which sequencing can resume; it +does not need to catch the moving L1 tip or preserve prior soft confirmations. +The [cockroach recovery guide](docs/recovery/cockroach.md) owns the checkpoint +requirements, stopping boundary, and rebuild procedure. + +Before changing recovery code, read its guide and both current bounded TLA+ +models. The automatic recovery guide's [Formal Verification](docs/recovery/README.md#formal-verification) +section explains their scopes; the models are not a proof of every recovery path. ## Threat Model (brief) @@ -134,13 +136,19 @@ See [`docs/threat-model/README.md`](docs/threat-model/README.md) for the full mo - **Trusted:** InputBox contract, our own Ethereum node (fail-stop, not byzantine), operator config, batch-submitter key. - **Adversarial:** `POST /tx` callers, direct-input senders, the L1 mempool and block builders (zombie transactions are a first-class threat). - **RPC endpoint:** single (`CARTESI_SEQUENCER_BLOCKCHAIN_HTTP_ENDPOINT`), trusted fail-stop, **must be one consistent node** — no fallback tier exists yet (see the threat model's actor table). -- **Self-trust:** the sequencer trusts its own code is correct. Bugs that emit malformed batches are fault states requiring manual intervention, not threats to defend against at runtime. +- **Self-trust:** normal operation assumes the sequencer's own code is correct. + Invariant violations fail loud. Bug-induced malformed batches require fixing + the bug and, when rebuilding is necessary, manual cockroach recovery; automatic + recovery does not repair software defects. - **In scope:** correctness bugs *and* exploitation. Under rollup semantics, a correctness bug that causes scheduler/sequencer state divergence is as severe as direct theft. ## Architecture Map Top-level layout follows the system's data flow. Each sequencer module corresponds to a writer role; the matching `storage/.rs` holds its storage half. +The implementation uses Rust edition 2024, Axum, SQLite (rusqlite/WAL), EIP-712 +signing, and SSZ batch encoding. + ### Workspace - `sequencer/` — sequencer **library** (no binary). App crates compose it into a binary. @@ -158,25 +166,17 @@ Top-level layout follows the system's data flow. Each sequencer module correspon ### Sequencer module layout -- `sequencer/src/lib.rs` — public sequencer API. The thin binary entrypoints live in `examples/wallet-sequencer/`. -- `sequencer/src/harness.rs` — CLI harness: the `setup`/`run`/`flush-mempool` subcommand parser, `dispatch`, and the exit-code projection. An app's `main` is ~5 lines (`run_main` + a genesis-app closure). -- `sequencer/src/http.rs` — shared HTTP error type, JSON `ErrorResponse`, `ApiConfig`, and `axum::serve` orchestration. -- `sequencer/src/commands/` — the operator command brackets: `setup` (phase A — pin identity, initial sync, genesis snapshot, atomic `setup_complete` fact), `run` (phase B — recover, prepare, admit, and boot workers; its `workers` supervisor lives beside it), and `flush` (`flush-mempool`). `sequencer/src/commands/` also owns the command-scoped `config` and `error` taxonomy (incl. the exit-code projection); `sequencer/src/runtime/` is exactly the runtime authority capabilities — the process lock and `shutdown` (runtime scope and graceful notification) — consumed crate-wide. `L1Config` lives in `sequencer/src/l1/`; the crate-wide wall clock is `sequencer/src/clock.rs`. -- `sequencer/src/ingress/` — public-facing HTTP + inclusion lane. - - `api.rs` — `POST /tx` and `GET /fee` handlers, JSON-rejection mapping. - - `inclusion_lane/` — single-lane hot-path loop (`mod.rs`), catch-up replay, config, error types. -- `sequencer/src/egress/` — internal read path. - - `api/` — `/ws/subscribe`, `/livez`, `/readyz`, `/healthz`. - - `l2_tx_feed/` — DB-backed ordered-tx feed. -- `sequencer/src/l1/` — L1 client surface. - - `reader.rs` — safe-input ingestion from InputBox into SQLite. - - `submitter/` — batch submitter (`worker.rs` + `poster.rs`); re-estimates fees every tick without carrying a fee floor from earlier attempts. The rationale, accepted liveness limits, and revisit criteria live in [`docs/l1-fee-policy.md`](docs/l1-fee-policy.md). - - `fee_oracle/` — setup-pinned L1 Uniswap V3 TWAP → `batch_policy.log_gas_price` (+ `log_gas_price_updated_at_ms`); fixed mode writes once at setup and has no worker. - - `eip1559.rs` — shared EIP-1559 fee estimation (poster, oracle, flusher). - - `provider.rs` — alloy provider construction. - - `partition.rs` — long-block-range retry helper. -- `sequencer/src/recovery/` — preemptive recovery startup procedure (`mod.rs`), runtime danger detector (`detector.rs`), and mempool flusher (`flusher.rs`). -- `sequencer/src/storage/` — SQLite persistence, split by writer role (`ingress`, `egress`, `l1_inputs`, `l1_submission`, `recovery`, `admin`, `safe_accepted_batches`, `snapshot_dumps`, plus shared `history`, `mod`, `open`, `convert`, `queries`, `mutations`, and `migrations/`). +Paths below are relative to `sequencer/src/`: + +- `lib.rs` and `harness.rs` — public API and shared CLI harness; app binaries supply their genesis-app constructor. +- `commands/` — `setup`, `run` (including worker supervision), and `flush`, with command configuration and exit-code classification. +- `runtime/` — exclusive process ownership and runtime scope/shutdown. +- `ingress/` — public HTTP handlers and the single inclusion lane. +- `egress/` — internal HTTP/WS API and DB-backed application-input feed. +- `l1/` — safe-input reader, batch submitter, fee oracle, shared EIP-1559 estimation, and provider access. +- `recovery/` — automatic startup repair, danger detector, and mempool flusher; the manual rebuild lives in `commands/setup/`. +- `storage/` — each writer role's persistence operations, shared queries, and schema. +- `http.rs` and `clock.rs` — shared HTTP errors/server setup and the crate-wide wall clock. `L1Config` lives in `l1/`. ## Key Concepts @@ -185,8 +185,8 @@ Top-level layout follows the system's data flow. Each sequencer module correspon - **Batch** — list of frames posted on-chain as one L1 transaction (SSZ-encoded). - **Inclusion lane** — the single ordering lane, with a latency-critical user-op regime and a slower L1-reconciliation regime ([ADR mechanism 4](docs/plans/2026-08-authority-boundary-adr.md)); the only writer of open batch/frame state ([I17](docs/invariants.md)) and the system's execution bottleneck. - **Batch submitter** — stateless worker that bulk-submits all pending batches each tick. Nonces are assigned by storage (structural `parent.nonce + 1`) when batches are closed; the submitter just reads them. -- **Danger detector** — background worker that polls `Storage::check_danger` on a fixed cadence and exits with `RecoveryRequired` when any non-`Safe` danger status fires. Never writes to the DB; never talks to L1. Crashes the process so startup recovery or refusal can run. -- **Fee oracle** — setup pins either a fixed exponent or a reviewed Uniswap V3 WETH/X TWAP tuple into deployment identity, and writes the first `log_gas_price` (+ observation stamp) in both modes. Setup requires a successful live quote; `run` performs no fee-source read before recovery/admission. Fixed mode has no worker; Uniswap launches a lazy refresher that immediately attempts a quote, persists successes, and retains the last price while logging and retrying transient source failures. The stamp is telemetry, not a runtime-admission or expiry gate. A shared-endpoint outage/stale view is already detected from L1 safe-head progress; a fee-source-only outage is an accepted economic residual (stale-low may subsidize DA, stale-high may reject users), not a canonical-correctness fault. Deterministic source misconfiguration, fatal arithmetic, and persistent storage faults remain terminal. The 10× margin lives in `batch_policy.log_slack`; it is a buffer rather than a bound on market movement, and frame fees stay immutable until the next frame opens. +- **Danger detector** — polls `Storage::check_danger` and signals the process to stop so startup can recover or refuse. It reads local facts; it never writes the DB or talks to L1. +- **Fee oracle** — setup pins and bootstraps a fixed price or Uniswap V3 TWAP source. The price informs future frame fees; an oracle-only outage is an accepted economic risk. The [threat model's actor table](docs/threat-model/README.md#actors-and-trust) owns the source assumptions and failure policy. - **Input reader** — ingests safe inputs from L1 InputBox and maintains the durable safe head, accepted-batch projection, and divergence marker in one atomic transaction (`sequencer/src/storage/l1_inputs.rs`); it hands the lane no in-memory cursor. - **L2 tx feed** — DB-backed application-input stream. HTTP snapshot headers provide `(EraId, RecoveryGeneration, ExecutedInputCount)`; WS validates that @@ -211,8 +211,13 @@ Top-level layout follows the system's data flow. Each sequencer module correspon - Rejections (`InvalidNonce`, `InvalidMaxFee`, `InsufficientFeeBalance`) produce no state mutation and are not persisted. These are protocol-level rejection semantics every app must implement: nonces prevent user-op replay, fees prevent spam against the sequencer's DA budget. ("Fee", not "gas" — the fee tracks DA; compute metering, if it ever exists, is a separate future concept.) - Included txs are persisted as frame/batch data in `batches`, `frames`, `user_ops`, `safe_inputs`, and `application_inputs`. Recovery metadata lives in `safe_accepted_batches`; batch lifecycle state (sealed/invalidated) lives on the `batches` row itself as write-once timestamps. - Frame fee is persisted in `frames.fee` and is fixed for the lifetime of that frame. The next frame's fee is currently sampled from `batch_policy_derived.recommended_fee` at rotation; oracle bootstrap writes the price before any Tip can sample it, and `log_slack` applies the 10× margin in log space. This is present behavior, not a reason for the five-block clock policy; hoisting fee to the batch is a later design with its own trade-offs. -- Wallet state (balances, nonces) is in-memory today — not persisted. -- **EIP-712 domain fields:** `name`, `version`, `chainId`, `verifyingContract`. `chainId` and `verifyingContract` come from `CARTESI_SEQUENCER_BLOCKCHAIN_ID` and `CARTESI_SEQUENCER_APP_ADDRESS` (validated against the RPC chain id at startup). All four fields must be present on both sides — both the sequencer and the on-chain scheduler construct the domain via `sequencer_core::build_input_domain`, the canonical shared constructor. +- Wallet balances and nonces live in memory between checkpoints; restart restores + a dump and replays persisted application inputs. +- **EIP-712 domain fields:** `name`, `version`, `chainId`, `verifyingContract`. + Setup pins the chain id and app address from `CARTESI_SEQUENCER_BLOCKCHAIN_ID` + and `CARTESI_SEQUENCER_APP_ADDRESS`, validating the chain id against RPC. All + four fields must be present on both sides; the sequencer and canonical + scheduler share `sequencer_core::build_input_domain`. ### InputBox payload classification @@ -244,41 +249,23 @@ User ops are executed only through `sequencer_core::application::validate_and_ex `Application` requires `Send`, with neither `Clone` nor `Sync`. Dumps must be durable and immutable, and restored engines must remain independent after source deletion. The opaque app prefix may be a file or directory; checkpoint disposal uses ordinary recursive filesystem deletion. Canonical inspection belongs to the separate `CanonicalState` trait; the native sequencer serves the comparison file in the checkpoint. The [C binding guide](docs/protocol/c-application-binding.md) maps the contract to native engines. -## Hot-Path Invariants - -The hot-path rules are owned elsewhere; this section is only the map. - -- Drain attribution, frame-clock monotonicity, the - content-identity check and the divergence freeze, history metadata, the - `WriteHead` cache, and the application-input sequence are registered in - [`docs/invariants.md`](docs/invariants.md) (the fail-loud check policy plus - I2, I3, I9, I10, I12–I18, I20) — that register owns them; do not restate - them here. -- Command admission, terminal stop, the two-regime lane and its - acknowledgement rule, and role-local authority are owned by the - [authority-boundary ADR](docs/plans/2026-08-authority-boundary-adr.md) - (mechanisms 1, 2, and 4); startup recovery by - [`docs/recovery/README.md`](docs/recovery/README.md). -- The frame-clock policy (five newly-safe blocks, one frame at the observed - tip, never interpolated, and its revisit trigger) is owned by - [`docs/protocol/scheduler-semantics.md`](docs/protocol/scheduler-semantics.md); - the no-preemption digestibility assumption by - [`docs/protocol/application-contract.md` §5](docs/protocol/application-contract.md#5-operational-capacity-for-l1-reconciliation). -- Queue admission (`429 OVERLOADED`), batch closure, and every other API or - storage-model shape are owned by [`README.md`](README.md). - -One rule lives here because nothing else owns it: preserve single-lane -deterministic ordering. Do not introduce extra concurrency in hot-path -ordering logic without explicit approval. - -## Storage Invariants - -Owned by [`docs/invariants.md`](docs/invariants.md): the writer-role table -(one writer role per fact), the `valid_*` view rule, `WriteHead` coherence -(I17), history metadata (I18), the application-input sequence and its canonical offsets (I10, I20). The schema -(`sequencer/src/storage/migrations/0001_schema.sql`) owns the write-once -batch lifecycle, the Tip's uniqueness, and the user-op identity rule. Do not -restate them here. +## Ordering and Storage + +Preserve single-lane deterministic ordering. Do not introduce extra concurrency +in hot-path ordering logic without explicit approval. + +The inclusion lane combines a latency-critical user-op regime with complete L1 +reconciliation. The [authority-boundary ADR](docs/plans/2026-08-authority-boundary-adr.md) +owns that split, acknowledgement rules, runtime ownership, and command admission. +The [scheduler contract](docs/protocol/scheduler-semantics.md#sequencer-frame-clock-policy) +owns the five-safe-block frame clock; the [Application contract](docs/protocol/application-contract.md#5-operational-capacity-for-l1-reconciliation) +owns the assumption that accumulated directs are digestible without preemption. + +Storage changes cross writer boundaries even when the SQL looks local. Read the +[invariant register](docs/invariants.md) for writer ownership, `valid_*` reads, +drain attribution, the content-identity/divergence freeze, `WriteHead` coherence, +and application-history offsets. The [schema](sequencer/src/storage/migrations/0001_schema.sql) +enforces write-once batch lifecycle, Tip uniqueness, and user-op identity. ## Type Boundaries @@ -294,41 +281,35 @@ restate them here. ## HTTP Endpoints - **Ingress** (public-facing): `POST /tx`, `GET /fee`. -- **Egress** (internal indexers/watchdog): `GET /ws/subscribe`, `GET /finalized_state`, `GET /finalized_state/inclusion_block`, `GET /latest_snapshot`, `GET /finalized_snapshot`, `GET /livez`, `GET /readyz`, `GET /healthz`. The snapshot/state endpoints are **operator-only** (no auth) and must not be exposed publicly; the streaming routes hold a GC lease for the response lifetime ([`docs/snapshots/lifecycle.md`](docs/snapshots/lifecycle.md)). +- **Egress** (internal indexers/watchdog): application-input subscriptions, + snapshot/state downloads, and health probes. Snapshot/state endpoints have no + authentication and **must not be exposed publicly**. Downloads hold a GC lease + for their response lifetime ([snapshot lifecycle](docs/snapshots/lifecycle.md)). Today both sides serve from one listener; the planned API split puts each side on its own port (same binary) so internal probes and subscribers can be firewalled from public submit traffic. -Message shapes, caps, close codes, and health semantics are **owned by [`README.md`](README.md)** (the API contract) — do not restate them here. +The [README API contract](README.md#api) owns routes, message shapes, caps, +close codes, and health semantics. -## Environment Variables +## Command Configuration -Split by subcommand (the phase split). **`setup`** (required): +Configuration follows the command phases: -- `CARTESI_SEQUENCER_BLOCKCHAIN_HTTP_ENDPOINT` -- `CARTESI_SEQUENCER_BLOCKCHAIN_ID` -- `CARTESI_SEQUENCER_APP_ADDRESS` -- `CARTESI_SEQUENCER_BATCH_SUBMITTER_ADDRESS` (the submitter address — `setup` is L1-read-only and never signs). **Must be a dedicated address**: `setup`'s detection gate refuses if the submitter's wallet nonce is unsettled, so reusing a busy address (e.g. the contract deployer, whose deploy-tx tail isn't safe at setup time) false-positives. The devnet uses anvil account 9 (`DEVNET_SEQUENCER_ADDRESS`), distinct from the account-0 deployer. -- `CARTESI_SEQUENCER_CHECKPOINT_BLOCK` (optional, default `0` = genesis) — the trusted checkpoint machine's L1 inclusion block. `setup` refuses (typed `SetupRefuse`, exit 40 = run `setup --recovery`) if a previous instance left work past it; plain `setup` detects only, and loading a non-genesis checkpoint machine is `setup --recovery`. +- Plain **`setup`** is L1-read-only and never signs. It pins chain/app/submitter + identity and the fee source. **`setup --recovery`** also needs the submitter + key to flush its outstanding transactions; see the [rebuild guide](docs/recovery/cockroach.md). +- **`run`** takes the RPC endpoint and signing key (or key file). It reads the + pinned chain id, app address, and submitter address from the database. -**`run`** (required) — chain id / app address / submitter address are read from the DB `setup` pinned, not from args: +**Use a dedicated submitter address.** Plain setup refuses an unsettled wallet +nonce, so sharing a busy contract-deployer address can trip the detection gate +while its deployment transactions are not yet safe. The devnet uses Anvil +account 9 (`DEVNET_SEQUENCER_ADDRESS`), separate from the account-0 deployer. -- `CARTESI_SEQUENCER_BLOCKCHAIN_HTTP_ENDPOINT` -- `CARTESI_SEQUENCER_AUTH_PRIVATE_KEY` or `CARTESI_SEQUENCER_AUTH_PRIVATE_KEY_FILE` - -**Optional** (names only — defaults and semantics are **owned by -[`sequencer/src/commands/config.rs`](sequencer/src/commands/config.rs)**; a -defaults list here drifted once already): `CARTESI_SEQUENCER_HTTP_ADDR`, `CARTESI_SEQUENCER_DATA_DIR`, -`CARTESI_SEQUENCER_LONG_BLOCK_RANGE_ERROR_CODES`, `CARTESI_SEQUENCER_BATCH_SUBMITTER_IDLE_POLL_INTERVAL_MS`, -`CARTESI_SEQUENCER_BATCH_SUBMITTER_CONFIRMATION_DEPTH`, `CARTESI_SEQUENCER_PREEMPTIVE_MARGIN_BLOCKS`, -`CARTESI_SEQUENCER_L1_READ_STALE_AFTER_BLOCKS` (fixed default, independent of the margin; -must be strictly below the danger threshold or startup refuses), -`CARTESI_SEQUENCER_SECONDS_PER_BLOCK`, and the runtime-only -`CARTESI_SEQUENCER_FEE_ORACLE_POLL_INTERVAL_MS`. Setup-only fee-oracle source knobs are -(`CARTESI_SEQUENCER_FEE_ORACLE_FIXED_LOG_GAS_PRICE`, -`CARTESI_SEQUENCER_FEE_TOKEN_ADDRESS`, -`CARTESI_SEQUENCER_WETH_ADDRESS`, `CARTESI_SEQUENCER_UNISWAP_V3_POOL`, -`CARTESI_SEQUENCER_FEE_ORACLE_TWAP_WINDOW_SECS` — mainnet/Sepolia default to - pinned USDC pool presets; other chains require an explicit source). +The [Running guide](README.md#running) gives invocation examples; +[`commands/config.rs`](sequencer/src/commands/config.rs) owns environment-variable +names, defaults, and validation. Check configuration there before documenting or +changing it, including checkpoint and fee-source selection. ## Coding Conventions @@ -343,23 +324,33 @@ must be strictly below the danger threshold or startup refuses), ## Documentation Practice -The corpus has two tenses, kept strictly apart: - -- **Living docs are timeless.** This file, `README.md`, `docs/protocol/`, - `docs/invariants.md`, `docs/recovery/`, `docs/snapshots/`, - `docs/threat-model/`, `docs/watchdog/`, and `docs/plans/` describe what is - true now and why — present tense, reasoning inline, no dates, no amendment - banners, no review codenames, no "previously/no longer". Each doc owns its - topic; others point at it rather than restating it. -- **History lives only in `docs/review/` and commit messages.** A review - ledger is append-only while its review is open. When it closes, distill it: - promote conclusions into the living docs, record settled decisions and - refuted proposals in [`docs/review/register.md`](docs/review/register.md), - and delete the process narration. Conclusions with reasoning outlive the - path taken to them. -- **Record deliberate absence once**, at the seam where someone would re-add - the mechanism, phrased as a positive design statement with its reason — - never as removal notices scattered across documents. +Write for a reader building a mental model. Start with the purpose, input, +result, and governing constraint; introduce implementation detail when it +explains a necessary behavior. Keep an intentional baseline here so important +concepts are discoverable before anyone knows to ask about them. Summaries +should point to the owner of a contract rather than becoming a second copy. + +Keep three kinds of material distinct: + +- **Current contracts and designs** describe what holds now and why. Use present + tense and reasoning inline, without amendment banners, review codenames, or + "previously/no longer" narration. Each topic has one owner. Some current + architecture documents live in `docs/plans/`; the directory name does not + make their established contracts optional. +- **Active plans** name open decisions, dependencies, and remaining work. They + must distinguish proposed behavior from implemented contracts. On completion, + put the durable design in its owner and reduce the plan to its remaining work + and links. +- **Historical evidence** lives in review ledgers, explicitly marked historical + documents, and commit history. A review ledger is append-only while open. + When it closes, promote conclusions into current docs, record settled and + refuted proposals in the [review register](docs/review/register.md), and remove + process narration. Retained superseded proposals must say they are historical + and link to the current contract; they are evidence, not instructions. + +**Record deliberate absence once**, at the seam where someone would re-add the +mechanism, phrased as a positive design statement with its reason. Avoid removal +notices scattered across documents. ## Testing Guidance @@ -374,34 +365,30 @@ Focus tests on: Prefer black-box tests around `POST /tx` and commit outcomes for integration. -Some `sequencer` tests use Anvil (Foundry). They run by default and fail with a clear message if `anvil` is not on PATH. Install Foundry or use `nix develop`. +Some `sequencer` tests use Anvil (Foundry). They run by default and fail with a +clear message if `anvil` is not on PATH. Use the configured Nix/direnv environment +or install Foundry. `canonical-test` additionally needs libslirp. -## Fast Start Commands +## Shell and Commands -See [`CLAUDE.md`](CLAUDE.md) for shell setup and the full command list. In short: +Use the configured Nix/direnv environment for Foundry, TLA+, and other project +tools. For noninteractive commands, prefer `direnv exec . `. Rust is +pinned to **1.95.0** in [`rust-toolchain.toml`](rust-toolchain.toml); verify both +`cargo --version` and `rustc --version` in the environment you use. A Nix-provided +Cargo or rustc may bypass rustup, so direnv alone does not guarantee the pinned +compiler is selected. Correct the toolchain selection before interpreting build +or dependency errors. ```bash -cargo check -cargo test --workspace --exclude canonical-test -cargo fmt --all -cargo clippy --all-targets --all-features -- -D warnings +direnv exec . cargo check +direnv exec . cargo test --workspace --exclude canonical-test +direnv exec . cargo test -p sequencer --lib # includes Anvil-backed tests +direnv exec . cargo fmt --all +direnv exec . cargo clippy --all-targets --all-features -- -D warnings ``` -Run server (two phases — `setup` once, then `run`; see `README.md` "Running"): - -```bash -# setup (L1-read-only; takes the submitter ADDRESS, not the key) -CARTESI_SEQUENCER_BLOCKCHAIN_HTTP_ENDPOINT=http://127.0.0.1:8545 \ -CARTESI_SEQUENCER_BLOCKCHAIN_ID=31337 \ -CARTESI_SEQUENCER_APP_ADDRESS=0x1111111111111111111111111111111111111111 \ -CARTESI_SEQUENCER_BATCH_SUBMITTER_ADDRESS=0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266 \ -cargo run -p wallet-sequencer -- setup - -# run (keyed; reads identity from the set-up DB) -CARTESI_SEQUENCER_BLOCKCHAIN_HTTP_ENDPOINT=http://127.0.0.1:8545 \ -CARTESI_SEQUENCER_AUTH_PRIVATE_KEY=0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80 \ -cargo run -p wallet-sequencer -- run -``` +The shared command harness is in [`sequencer/src/harness.rs`](sequencer/src/harness.rs). +See [Running](README.md#running) for the two-phase `setup` / `run` workflow. ## Always / Ask First / Never @@ -411,14 +398,15 @@ cargo run -p wallet-sequencer -- run - Preserve API error shape and status code mapping unless intentionally changing the API contract. - Add or update tests when logic changes. - Run at least `cargo check` before finishing. -- Read `docs/recovery/` before touching recovery code, and `docs/threat-model/` before touching trust-boundary code. +- Read the relevant recovery guide and both current TLA+ models before touching + recovery code, and the threat model before touching trust-boundary code. - Check [`docs/invariants.md`](docs/invariants.md) before changing anything it lists as load-bearing, and [`docs/review/register.md`](docs/review/register.md) for open findings in the code you're about to touch and for decisions already settled or refuted. ### Ask First - Changing tx wire format (`UserOp`, SSZ payload layout, EIP-712 domain fields). - Changing DB schema or migration strategy. -- Altering rejection semantics (what consumes nonce/gas vs what is rejected). +- Altering rejection semantics (what consumes nonce/fee vs what is rejected). - Introducing concurrency changes to commit ordering. - Changing chunk/frame/batch closure or ack semantics. @@ -444,18 +432,23 @@ Before finishing a change, ensure: 3. Formatting and lints are clean, or list any unresolved warnings explicitly. 4. PR summary includes **what changed**, **why it changed**, and **risk / compatibility notes**. -## Related Documents - -- [`README.md`](README.md) — product framing, user-facing trust model, **API contract** (endpoint shapes, caps, close codes, health semantics). -- [`CLAUDE.md`](CLAUDE.md) — shell setup, quick reference, pointer back here. -- [`docs/protocol/`](docs/protocol/) — the authoritative protocol contracts: [`scheduler-semantics.md`](docs/protocol/scheduler-semantics.md) (the canonical acceptance algorithm, I1), [`application-contract.md`](docs/protocol/application-contract.md) (the `Application` trait contract), and [`c-application-binding.md`](docs/protocol/c-application-binding.md) (the native C binding). -- [`docs/invariants.md`](docs/invariants.md) — register of cross-module invariants (what's load-bearing across files) + the fail-loud check policy. -- [`docs/review/register.md`](docs/review/register.md) — the review register: open findings, settled decisions, refuted proposals (do-not-re-propose), and the review history table; the dated ledgers beside it carry the evidence the table points at. -- [`docs/plans/`](docs/plans/) — the architecture decision record ([`2026-08-authority-boundary-adr.md`](docs/plans/2026-08-authority-boundary-adr.md)), active coordination tracks, and in-flight design handoffs. -- [`docs/threat-model/README.md`](docs/threat-model/README.md) — trust boundaries, in-scope and out-of-scope threats. -- [`docs/recovery/README.md`](docs/recovery/README.md) — recovery design, TLA+ formal verification, design history. -- [`docs/snapshots/`](docs/snapshots/) — app snapshots: [`format.md`](docs/snapshots/format.md) (dump trait + wire format) and [`lifecycle.md`](docs/snapshots/lifecycle.md) (creation/acceptance/GC/lease design + crash-safety). -- [`docs/watchdog/operator-deployment.md`](docs/watchdog/operator-deployment.md) — production-like watchdog (Sepolia / mainnet; internal snapshot API). -- [`docs/watchdog/getting-started.md`](docs/watchdog/getting-started.md) — local dev: watchdog + `sequencer-devnet` on Anvil. -- [`docs/watchdog/README.md`](docs/watchdog/README.md) — watchdog architecture, compare vs advance modes, test commands. -- [`sequencer-core/`](sequencer-core/) — shared domain types and protocol contracts. +## Reading Routes + +Follow the rows that intersect the change. Each destination explains the +cross-module consequences to check before editing; follow its links when the +work reaches another boundary. + +| Work | Read first and why | +|---|---| +| Scheduler acceptance, batch nonces, direct-input ordering, or frame clock | [Scheduler semantics](docs/protocol/scheduler-semantics.md) — canonical algorithm and the implementations that must agree. | +| Inclusion lane, storage writes, or runtime concurrency | [Invariant register](docs/invariants.md) — enforcement and consumers; [authority-boundary ADR](docs/plans/2026-08-authority-boundary-adr.md) — ownership, acknowledgement, admission, and terminal stop. | +| Application implementation, execution, or native integration | [Application contract](docs/protocol/application-contract.md) — determinism, progress, failure, capacity, and checkpoints; [C binding](docs/protocol/c-application-binding.md) for native engines. | +| Automatic recovery or danger detection | [Automatic recovery](docs/recovery/README.md), then [preemptive.tla](docs/recovery/preemptive.tla) and [admission.tla](docs/recovery/admission.tla) — repair ordering and the models' bounded guarantees. | +| Manual rebuild after lost state or a sequencer bug | [Cockroach recovery](docs/recovery/cockroach.md) — trusted checkpoint, fixed input boundary, and fresh baseline. | +| API, subscriber replay, or application-history coordinates | [README API contract](README.md#api) — wire behavior; [application history](docs/plans/application-history.md) — era, generation, offsets, and recovery boundaries. | +| Snapshots, restart, export, retention, or watchdog checkpoints | [Snapshot lifecycle](docs/snapshots/lifecycle.md) — durable publication, accepted comparison points, leases, and GC; [wallet format](docs/snapshots/format.md) when changing wallet bytes. | +| Trust boundaries, provider behavior, or hostile L1 input | [Threat model](docs/threat-model/README.md) — actor assumptions, supported failures, and residual risks. | +| Submission fees or oracle pricing | [L1 fee policy](docs/l1-fee-policy.md) — estimation and replacement limits; [threat-model actor table](docs/threat-model/README.md#actors-and-trust) — oracle source and outage assumptions. | +| Command setup or deployment configuration | [Running](README.md#running) and [config.rs](sequencer/src/commands/config.rs) — invocation, identity pinning, defaults, and validation. | +| Watchdog development or operation | [Architecture](docs/watchdog/README.md); [local dev](docs/watchdog/getting-started.md) for Anvil; [operator deployment](docs/watchdog/operator-deployment.md) for Sepolia/mainnet. | +| A new mechanism, simplification, or work spanning an active track | [Review register](docs/review/register.md) — relevant open findings and settled/refuted reasoning; [coordination tracks](docs/plans/2026-07-coordination-tracks.md) — remaining work and dependencies. Consult dated evidence when its reasoning is needed. | diff --git a/CLAUDE.md b/CLAUDE.md index 037795a4..a10e1d35 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,73 +1,7 @@ # CLAUDE.md -Quick reference for working in this repository. For the full guide — architecture, duality, recovery, invariants, threat model, and rules — read [`AGENTS.md`](AGENTS.md). +Read [`AGENTS.md`](AGENTS.md) before working in this repository. It contains the +shared mental model, safety constraints, and contribution rules for all agents. -## Shell Environment - -This project uses Nix + direnv. Before running any command that needs project tools (Foundry, TLA+, etc.), activate the direnv environment: - -```bash -eval "$(direnv export bash 2>/dev/null)" -``` - -This makes `anvil`, `forge`, `cast`, `tlc`, and other Nix-provided tools available. Cargo and rustc are available without direnv. - -## Commands - -```bash -cargo check # compile check -cargo test --workspace --exclude canonical-test # run tests (canonical-test needs libslirp) -cargo fmt --all # format -cargo clippy --all-targets --all-features -- -D warnings # lint -cargo test -p sequencer --lib # includes Anvil-backed tests (needs Foundry on PATH) -``` - -## What This Is - -Off-chain sequencer for an app-specific DeFi rollup. Accepts signed user operations, issues low-latency soft confirmations, and posts batches to L1. Currently backed by a placeholder wallet app (transfer, withdrawal). **Security-critical infrastructure** — handle every change accordingly. - -Rust edition 2024 / Axum API / SQLite (rusqlite, WAL) / EIP-712 signing / SSZ encoding. - -## Workspace Layout - -- `sequencer/` — sequencer library (no binary; app crates build the binary). -- `sequencer-core/` — shared domain types consumed by both sequencer and scheduler. -- `examples/app-core/` — placeholder wallet app implementing `Application`. -- `examples/wallet-sequencer/` — binary crate: wallet app + sequencer library. -- `bindings/c-app-engine/` — reusable native engine adapter implementing `Application` through a C ABI. -- `bindings/c-app-sequencer/` — optional C-engine CLI host and external-archive binary. -- `examples/c-wallet-engine/` — reference C ABI exports, genesis tool, and conformance tests. -- `examples/c-wallet-sequencer/` — binary composing the C-engine host with the reference wallet engine. -- `examples/canonical-app/` — on-chain scheduler reference implementation. -- `examples/canonical-test/` — e2e test harness for the canonical app. -- `sdk/rust-client/` — Rust client library for the sequencer API. -- `tests/{benchmarks,e2e,harness}/` — test infrastructure. - -## Sequencer Module Layout - -`sequencer/src/` is organized by writer role; `storage/.rs` holds each role's storage half. - -- `commands/` — the operator command brackets (`run/` plus its worker - supervisor, `setup/`, `flush`) and their command-scoped `config` and - `error` taxonomy (incl. exit-code projection). -- `runtime/` — the runtime authority capabilities, consumed crate-wide: - the exclusive process lock and the runtime scope/shutdown machinery. -- `ingress/` — public-facing: `api.rs` (`POST /tx`, `GET /fee`) + `inclusion_lane/` (hot path). -- `egress/` — internal read path: `api/` (WS subscribe + health) + `l2_tx_feed/`. -- `l1/` — reader, submitter, fee oracle, provider, partition helper. -- `recovery/` — startup preemptive-recovery procedure, runtime danger detector, mempool flusher. -- `storage/` — SQLite persistence, split per writer role. -- `http.rs` — shared HTTP error type + `axum::serve` orchestration; `clock.rs` — the crate-wide wall clock. - -## Before You Start Real Work - -- **[`AGENTS.md`](AGENTS.md)** — mission, requirements, invariants, duality, recovery, conventions, rules. -- **[`docs/protocol/`](docs/protocol/)** — the authoritative protocol contracts: [`scheduler-semantics.md`](docs/protocol/scheduler-semantics.md) (canonical acceptance algorithm), [`application-contract.md`](docs/protocol/application-contract.md) (the `Application` trait), and [`c-application-binding.md`](docs/protocol/c-application-binding.md) (the native C binding). Read before touching the scheduler, the gold frontier, the fold, or an `Application` impl. -- **[`docs/invariants.md`](docs/invariants.md)** — cross-module invariants register + the fail-loud check policy. Check it before changing anything it lists as load-bearing. -- **[`docs/review/register.md`](docs/review/register.md)** — the review register: open findings, settled decisions, refuted proposals (do-not-re-propose). Check it for open findings in code you're about to touch, and before proposing a mechanism or simplification. -- **[`docs/plans/`](docs/plans/)** — the [authority-boundary ADR](docs/plans/2026-08-authority-boundary-adr.md), active coordination tracks, and in-flight design handoffs. Check before starting work that might belong to a track. -- **[`docs/threat-model/README.md`](docs/threat-model/README.md)** — trust boundaries and in-scope threats. -- **[`docs/recovery/README.md`](docs/recovery/README.md)** — preemptive recovery design + TLA+ proofs. -- **[`docs/snapshots/lifecycle.md`](docs/snapshots/lifecycle.md)** — snapshot lifecycle design + invariants (take/promote/GC, crash-safety). Read before touching the inclusion lane's safe-frontier/snapshot path. -- **[`docs/watchdog/operator-deployment.md`](docs/watchdog/operator-deployment.md)** — watchdog on live L1 (Sepolia / mainnet, production-like). -- **[`docs/watchdog/getting-started.md`](docs/watchdog/getting-started.md)** — local dev: watchdog + `sequencer-devnet` on Anvil. +- [Shell and commands](AGENTS.md#shell-and-commands) — toolchain selection and validation commands. +- [Reading routes](AGENTS.md#reading-routes) — the contracts to read for the work at hand. diff --git a/README.md b/README.md index 3df30f2d..68c1a0a9 100644 --- a/README.md +++ b/README.md @@ -26,9 +26,9 @@ Sequencer (off-chain) Scheduler (on-chain) ``` When things go well, the sequencer's chain and the scheduler's view converge. -When batches are becoming stale on L1, the sequencer detects the doomed suffix -and runs standard recovery. Terminal canonical divergence is the distinct -content-identity case below. +When batches risk becoming stale on L1, the sequencer stops serving and startup +determines the required repair. A lost or untrustworthy local state instead +requires an operator rebuild. Both recovery modes are described below. ## Trust Model @@ -60,9 +60,24 @@ backstop, not proof that arbitrary application or scheduler divergence cannot exist, and it does not replace the watchdog. The mechanism and its bounds are recorded in [`docs/invariants.md`](docs/invariants.md) (I9 and I15). -The third case is handled by the recovery subsystem. Batches that are too old when they reach L1 (`inclusion_block − safe_block ≥ MAX_WAIT_BLOCKS`) are skipped by the scheduler. This "staleness" poisons the nonce counter: all subsequent batches become unreachable regardless of their individual freshness. The sequencer detects this via a danger-zone threshold, preemptively goes offline, flushes the L1 mempool, and cascade-invalidates the doomed chain. See [`docs/recovery/`](docs/recovery/) for the full design, TLA+ formal verification, and design history. +## Recovery -The sequencer trusts its own code is bug-free. Recovery means recovery from liveness failures, which can legitimately happen even in the absence of bugs (infrastructure outages, network failures, gateway failure). Code-level bugs are a separate problem handled by tests and review. See [`docs/threat-model/README.md`](docs/threat-model/README.md) for the complete threat model applied across the codebase. +**[Standard recovery](docs/recovery/README.md)** runs automatically at startup +using the existing database. It handles liveness failures such as outages and +extended downtime: reconcile L1 outcomes, invalidate the affected optimistic +suffix, and resume from retained state. Stale batches do not consume the +scheduler's expected nonce, so their successors cannot be accepted until recovery +supplies a replacement at that nonce. + +**[Cockroach recovery](docs/recovery/cockroach.md)** is an operator-triggered +rebuild when the local database is lost or cannot be trusted. This includes a +sequencer bug that corrupted state or emitted malformed batches: fix the bug, +choose a trusted canonical application checkpoint, then rebuild in a fresh data +directory. The command processes historical L1 inputs through the canonical +scheduler and prepares a baseline for resuming normal operation. + +The [threat model](docs/threat-model/README.md#self-trust) explains the boundary +between normal operation's self-trust and manual repair after a bug. ## Failure Modes @@ -70,7 +85,7 @@ The sequencer is designed to handle: - **L1 provider outages** — workers retry with exponential backoff. The inclusion lane and API continue operating locally. A wall-clock fallback detects when an outage pushes batches into the danger zone. - **Undiagnosed interruptions (OOM, SIGKILL, reboot)** — restart can recover automatically: every boot derives any required recovery from SQLite and L1 safe state through startup recovery, never assuming the previous exit was clean. Terminal errors returned through a command bracket best-effort record their cause in `terminal_faults`; terminal runtime aborts leave only process diagnostics. -- **Extended downtime** — startup syncs to the current L1 safe head, flushes if needed, and recovers before admission; restart policy is the exit-code contract (a terminal exit means: do not restart, page an operator — the one manual remedy is a fresh-directory `setup --recovery` after canonical divergence). +- **Extended downtime** — startup syncs to the current L1 safe head, flushes if needed, and recovers before admission. A terminal exit requires operator investigation; rebuilding untrustworthy state follows the cockroach recovery procedure above. - **Adversarial L1 mempool** — block builders and private mempools are treated as adversarial. The recovery flusher consumes every pending nonce slot with a no-op so delayed "zombie" submissions cannot land later. ## Interfaces @@ -91,11 +106,15 @@ The batch submitter posts closed batches to L1's InputBox contract. Each batch c The sequencer runs in two phases. **`setup`** pins the deployment identity (including the reviewed fee-oracle source), does the initial L1 sync, and registers the genesis -snapshot — run it once. It is L1-read-only: it takes the batch-submitter +snapshot — run it once. Plain `setup` is L1-read-only: it takes the batch-submitter *address*, never the signing key. **`run`** boots the sequencer from the set-up DB, reading identity from it (so chain id / app address are not `run` arguments); it holds the signing key because it submits. +For rebuilding from a trusted checkpoint, follow the +[cockroach recovery procedure](docs/recovery/cockroach.md#run-a-rebuild). +`setup --recovery` also needs the submitter key because it flushes transactions. + ```bash # Phase A — set up the data dir (run once; idempotent). CARTESI_SEQUENCER_BLOCKCHAIN_HTTP_ENDPOINT=http://127.0.0.1:8545 \ @@ -310,21 +329,18 @@ docker pull ghcr.io/cartesi/sequencer-watchdog:vX ## Development -```bash -cargo check # compile -cargo test --workspace --exclude canonical-test # test (canonical-test needs libslirp) -cargo fmt --all # format -cargo clippy --all-targets --all-features -- -D warnings # lint -``` - -Some tests require [Foundry](https://getfoundry.sh) (`anvil` on PATH). They run by default and fail with a clear message if unavailable. This project uses Nix + direnv for tooling — `direnv allow` provides Foundry, TLA+, and other dependencies. +The shared [development commands](AGENTS.md#shell-and-commands) cover Rust +toolchain selection, Nix/direnv tooling, compilation, tests, formatting, and +linting. Read the [testing guidance](AGENTS.md#testing-guidance) before choosing +validation for a change; some tests require Anvil or libslirp. ## Further Reading - [`AGENTS.md`](AGENTS.md) — developer guide: architecture, conventions, duality, recovery, invariants, rules. -- [`CLAUDE.md`](CLAUDE.md) — quick reference for shell setup and commands. +- [`CLAUDE.md`](CLAUDE.md) — Claude entrypoint to the shared agent guide. - [`docs/threat-model/README.md`](docs/threat-model/README.md) — trust boundaries, in-scope and out-of-scope threats. -- [`docs/recovery/README.md`](docs/recovery/README.md) — recovery design, TLA+ formal verification, design history. +- [`docs/recovery/README.md`](docs/recovery/README.md) — automatic recovery, TLA+ formal verification, design history. +- [`docs/recovery/cockroach.md`](docs/recovery/cockroach.md) — manual rebuild after lost state or a sequencer bug. - [`docs/watchdog/getting-started.md`](docs/watchdog/getting-started.md) — step-by-step: run the watchdog with a local sequencer. - [`docs/watchdog/operator-deployment.md`](docs/watchdog/operator-deployment.md) — watchdog on live L1 (Sepolia staging, mainnet production). - [`docs/watchdog/README.md`](docs/watchdog/README.md) — watchdog architecture, modules, and test commands. diff --git a/docs/recovery/README.md b/docs/recovery/README.md index 32dc38fd..6513d28e 100644 --- a/docs/recovery/README.md +++ b/docs/recovery/README.md @@ -1,8 +1,20 @@ # Batch Recovery -This document describes the recovery design for the sequencer: how the system detects that batches are failing to land on L1, how startup recovers to a consistent state, and where runtime authority begins. Two complementary bounded TLA+ models cover the design: [`preemptive.tla`](preemptive.tla) for batch/slot safety and [`admission.tla`](admission.tla) for startup phase ordering and admission. They do not currently model the external era/generation/base metadata or the canonical `application_inputs` projection or snapshot artifact/GC lifecycle; their crash atomicity is enforced by the SQLite transaction boundaries and schema triggers described below. - -See `AGENTS.md` "Batch Staleness and Recovery" for quick-reference tables and function names. +There are two recovery paths: + +- **Standard recovery**, described on this page, runs automatically at startup. + It uses the existing database to repair an optimistic suffix that can no + longer be relied on and decides whether the sequencer can resume serving. +- **[Cockroach recovery (`setup --recovery`)](cockroach.md)** is an operator-triggered + rebuild from a trusted canonical application checkpoint and L1. Use it after + database loss or unusable local state, including after fixing a sequencer bug. + +Two complementary bounded TLA+ models cover standard recovery: +[`preemptive.tla`](preemptive.tla) for batch/slot safety and +[`admission.tla`](admission.tla) for startup phase ordering and admission. They +do not currently model the external era/generation/base metadata, the canonical +`application_inputs` projection, or snapshot artifact/GC lifecycle; their crash +atomicity is enforced by SQLite transaction boundaries and schema triggers. ## Runtime lifecycle at a glance @@ -33,7 +45,8 @@ Batches form a tree where each node is a batch and edges point from child to par Batches have two identifiers: - **Index** (`batch_index`): monotonically increasing, unique, never reused. Creation order. -- **Nonce** (`batch_nonce`): depth of the node in the tree. Assigned by the batch submitter to valid closed batches. +- **Nonce** (`batch_nonce`): scheduler sequence number. Storage derives it as + `parent.nonce + 1`, or the deployment's anchor nonce for a parentless root. In normal operation the tree degenerates into a list -- index and nonce increase in lockstep. Branches appear only after recovery, when a suffix of the chain is invalidated and a new batch forks from the last valid ancestor. @@ -49,9 +62,11 @@ The implementation handles the nonce-0 case **structurally**: `open_fresh_tip_in #### Cockroach recovery generalizes the root nonce (the anchor) -Cockroach recovery (`setup --recovery`) rebuilds a wiped DB from a trusted checkpoint and must resume submitting at nonce `N'` without replaying history — so the rebuilt tree is rooted at `N'`, not 0. Rather than plant a fake "sentinel" batch at `N'-1`, the batch-tree anchor generalizes the structural root: a `batch_tree_anchor` singleton holds the nonce the parentless root carries (default `0`; recovery sets `N'`). The same `open_fresh_tip_in_tx` / `compute_next_nonce(parent = None)` path then roots `run`'s first tip at `N'`, and `trg_enforce_nonce_contiguity` validates the root against the anchor (exact match) instead of a hard-coded 0. There is **no sentinel batch row** — the root tip *is* the anchored batch. Normal deployments keep anchor `0` and are byte-identical. See [I16](../invariants.md) and the [cockroach-recovery design](#cockroach-recovery-setup---recovery) below. - -A sealed `N'-1` sentinel was considered and rejected: a valid closed batch at `N'-1` is a legal cascade pivot, so a runtime cascade could invalidate it and leave the tree re-rooting at 0 (ABORTed by the unchanged contiguity trigger) — an unguarded reliance on "the frontier never drops to `N'-1`". The anchor has no such hidden dependency. +A fresh deployment roots its batch tree at nonce 0. Cockroach recovery roots it +at the scheduler's next nonce after replay. The `batch_tree_anchor` preserves +that starting nonce even if a later cascade removes the whole local branch. +[I16](../invariants.md#i16-the-batch-tree-has-exactly-one-valid-parentless-root-carrying-the-deployments-anchor-nonce) +owns the root invariant and its enforcement. ## Coloring @@ -372,25 +387,15 @@ Dead batches occupy `w_nonce` slots strictly below `walletNonce`. Recovery batch ## Cockroach recovery (`setup --recovery`) -Everything above is **standard recovery**: the sequencer's own bookkeeping -(the batch tree, pending dumps) lets startup cascade a doomed suffix and -resume. The repair decision is automatic, not an operator-designed -reconstruction: recovery crosses a process boundary, and the next boot -inspects fresh facts regardless of how the prior process -died. Admission and the terminal-fault black box are owned by -[ADR mechanism 2](../plans/2026-08-authority-boundary-adr.md#2-fact-derived-admission-and-the-terminal-fault-black-box). - -**Cockroach recovery** is the catastrophe path — the local DB is lost or has diverged (`CanonicalDivergence`, [I15](../invariants.md)). There is no tree to cascade; the operator supplies a fresh or explicitly wiped data directory and rebuilds canonical logical state from a trusted checkpoint plus L1. It is an operator-driven, one-shot `setup` mode, not a runtime action. There is no automated DB replacement, clone detection, distributed fencing, or partial-fill resume state machine. The summary: - -Given a trusted checkpoint machine `S` at block `B` (a finalized `dumps//` dir, carrying `N` = its resume nonce and `A` = its last-executed safe block), `setup --recovery --checkpoint-block B --checkpoint-dump-dir ` runs **flush → fold → fill**: - -1. **Flush** the wallet nonce (keyed — recovery, unlike plain `setup`, signs) so every previous-instance batch resolves at safe depth `≤ C`, the post-flush safe head. Re-sync `safe_inputs` through `C`. -2. **Fold** (the pure `sequencer-core` engine, shared with the on-chain scheduler so it is consistent by construction): seed the fridge from the `(A, B]` directs (drop batches — already in `S`), replay the `(B, C]` stream, drain the leftover fridge at `C`. Yields `(S', N')` = the advanced app state and the resume nonce. -3. **Fill** a consistent DB: write the recovered application dump first, then atomically register its complete `(era, generation = 0, K, C)` history baseline, anchor `N'`, parentless root frame at `C`, snapshot, and `setup_complete`. The collapsed prefix creates no application rows. The first later application input has offset `K`; ordinary recovery falls back to immutable `C` if the root is invalidated. The terminal-drained baseline is a local restore point and is not automatically a canonical comparison checkpoint at `C`. - -During rebuild the accepted frontier is deferred until the baseline exists. The first `run` sync seeds expected nonce `N'` and scans only inputs after `C`, explicitly excluding the trusted prefix. Replaying the old prefix with a later expected nonce could reinterpret a rejected future-nonce batch as accepted. Checkpoint state and nonce remain operator-trusted; the export receipt checks metadata agreement rather than independently verifying the checkpoint. Rebuild is one-shot after completion. File-first creation plus atomic registration removes partial-baseline resume states; a failed transaction leaves only an orphan artifact. See [cockroach recovery](cockroach.md) for the full contract. +Use [cockroach recovery](cockroach.md) when the database is lost or local state +is unusable, including after a sequencer bug. Fix the bug first, then rebuild in +a fresh data directory from a trusted, sufficiently advanced canonical +application checkpoint and L1. The recovery guide owns checkpoint requirements, +the replay procedure, and the resulting resume baseline. -The detect-and-refuse gate is the *trigger*: a fresh `setup` that finds a previous instance's batches past the checkpoint refuses with exit `40` (`EXIT_SETUP_NEEDS_RECOVERY`), pointing the operator here. +Plain `setup` also directs the operator to this path when it detects a previous +instance's batches past the checkpoint: it refuses with exit `40` +(`EXIT_SETUP_NEEDS_RECOVERY`). ## Canonical divergence (terminal, outranks every arm) diff --git a/docs/recovery/cockroach.md b/docs/recovery/cockroach.md index f02510f0..0016e269 100644 --- a/docs/recovery/cockroach.md +++ b/docs/recovery/cockroach.md @@ -1,119 +1,131 @@ -# Cockroach recovery (`setup --recovery`) - -When the local DB is lost or has diverged, the operator rebuilds from a trusted -application checkpoint and L1 in a fresh data directory. This is a one-shot -setup operation. [Standard recovery](README.md) instead keeps the database and -invalidates an unaccepted suffix. - -The [canonical scheduler fold](../../sequencer-core/src/scheduler/fold.rs) -reconstructs the state. Its terminal drain also prepares the next local frame: -the result is a **resume baseline**, which can be ahead of canonical application -execution at the stopping block. It is not exposed as a finalized comparison -checkpoint merely because the L1 inputs used to construct it are safe. - -## Data dictionary - -| Symbol | Meaning | Source | -|---|---|---| -| `S` | Trusted application state at checkpoint block `B`. | Restored application dump. | -| `A` | Last executed application safe block in `S`; pending directs are seeded from `(A, B]`. | Application progress in the dump. | -| `B` | Checkpoint inclusion block. | Exported `checkpoint.toml`; must equal the configured checkpoint block. | -| `N` | Scheduler's next batch nonce at `B`. | Exported receipt, checked against immutable `info.toml`. | -| `C` | Post-flush safe stopping block. | Flusher result. | -| `N'` | Next batch nonce after folding through `C`; the new batch-tree anchor. | Fold result. | -| `K` | Application count after the terminal drain; first local history entry is `K`. | Recovered application progress. | -| `E`, `g` | New UUIDv4 era and generation zero. | Atomic completed-baseline registration. | - -The checkpoint contract requires `A < B`, checked at load. The sole exception -is the known empty genesis checkpoint (`B = 0`, next nonce and app count zero). -At a non-genesis `A = B`, a direct arriving after the accepted batch in block -`B` can still be pending; the empty `(A, B]` seed would silently omit it. -A recovery export carries a canonical application dump and a separate receipt; -baseline downloads and ordinary optimistic snapshots have no such receipt. - -### Trusted checkpoint boundary - -The application state, resume nonce, and relationship between the checkpoint and -L1 are operator-trusted. The receipt catches accidentally mixing an artifact, -nonce, or configured inclusion block; it does not independently verify state -against L1. A wrong checkpoint nonce, whether low or high, is outside the -supported model. The content-identity check verifies newly observed acceptance -after the baseline, not the opaque prefix or checkpoint correctness. - -An independent verification would need a trusted canonical-machine checkpoint -or replay from an independently trusted origin. The infrastructure subscriber's -application dump is not a substitute for that watchdog trust boundary. - -## The procedure: flush → fold → fill - -1. **Load the checkpoint.** Restore `S`, read both metadata files, verify their - nonce agreement and the configured `B`, then derive `A` and require `A < B` - or the known empty genesis checkpoint. -2. **Flush stranded transactions.** Consume unresolved wallet nonce slots and - wait for safe finality, obtaining `C`. The lost database cannot supply its - previous watermark, so the flush uses the provider's pool view. A dropped - transaction alive elsewhere can evade that view; a later accepted foreign or - mismatched landing after `C` freezes the new instance and requires another - rebuild. The trusted provider is fail-stop, not Byzantine. -3. **Re-sync raw L1 inputs.** The safe head `H1` must cover `C`; it can be later. - Acceptance projection is deferred while the new local tree is absent. -4. **Source disjoint fold ranges.** Seed external directs in `(A, B]`, then - replay all raw inputs in `(B, C]`. Sender classification excludes own batch - envelopes from the direct-input seed queue. -5. **Fold and drain.** The scheduler processes the stream with expected nonce - `N`, then drains every remaining direct through `C`, producing `(S', N')`. - A young direct still waiting in the canonical scheduler may therefore already - be present in `S'`. The resumed frame covers it before executing new user ops. -6. **Write the baseline artifact, then publish it.** First create and durably - sync the immutable dump. One SQLite transaction then creates history - `(E, 0, K, C)`, sets anchor `N'`, opens its parentless root frame at `C`, - registers the baseline artifact, and records `setup_complete`. It creates no - application-input rows for the collapsed prefix. - -On the first `run` sync, acceptance starts at `N'` and scans only raw inputs -whose block is **strictly greater than `C`**. Nonce filtering alone is unsound: -a previously rejected future-nonce batch inside the old prefix could match the -new expected nonce. The opaque prefix is never classified again. - -Inputs in `(C, H1]` remain available to the inclusion lane. Its next complete -reconciliation executes them once and records application entries beginning at -`K`. Raw L1 input indices and application offsets remain separate coordinates. - -## Recovery and retention - -`C` is the immutable fallback reconciliation boundary. While valid frames -survive, their latest `safe_block` gives the already-reconciled boundary. If -standard recovery invalidates the original root, it falls back to `C`, so -inputs represented by the baseline are never executed again. Canonical -application rows belonging to invalidated batches are deleted atomically with -the generation change and suffix invalidation. - -Startup loads the latest surviving batch-close snapshot, falling back to the -baseline. Admission requires a **rollback-safe checkpoint**: either that -baseline or a retained accepted batch snapshot. An optimistic snapshot alone -cannot satisfy this requirement because a cascade may discard its whole suffix. - -Once an accepted post-baseline batch snapshot exists, standard recovery cannot -invalidate it or return to the original baseline. GC can retire the baseline -artifact, subject to download leases. Immutable baseline metadata remains. -Snapshots with equal application counts remain distinct artifacts associated -with distinct batches; acceptance and retention never infer identity from count. - -## Crash-safety & idempotency - -A completed rebuild refuses another `setup --recovery`. Before completion, -there is no partially registered history or recovery root to resume: - -- A failure during artifact creation leaves setup incomplete. -- A failed registration transaction leaves neither baseline history, root, - anchor update, snapshot row, nor completion marker; any durable file is an - orphan for cleanup. -- A successful transaction establishes all those facts together. There are no - nullable baseline coordinates and no physical replay padding. - -Early identity pinning and raw L1 ingestion can survive an incomplete attempt. -They do not establish an application-history era. The process lock and setup -admission exclude runtime serving before the complete baseline exists. +# Cockroach recovery: rebuild from L1 + +Cockroach recovery (`setup --recovery`) creates a fresh sequencer starting state +from a trusted application checkpoint and historical L1 inputs. Use it when the +local database is lost or cannot be trusted, including after a sequencer bug has +corrupted its state or produced malformed batches. **Find and fix the bug before +rebuilding.** The operator initiates recovery; the command automates the rebuild. + +The procedure is **flush → fold → fill**: + +1. **Flush** outstanding submitter transactions and choose a fixed safe L1 + stopping block. +2. **Fold** the input history through the canonical scheduler, starting from the + trusted checkpoint. Every input receives its normal scheduler treatment: + accepted batches execute, malformed or rejected batches are skipped, and + direct inputs are queued and drained. Finally, drain every remaining direct + input through the stopping block. +3. **Fill** a fresh database with the recovered application state and next batch + nonce, ready for `run`. + +The result has accounted for the whole input prefix and carries no inherited +speculative user-operation suffix. It does not need to reach the moving tip: +normal operation handles inputs after the stopping block. + +This is a **resume baseline**. The final drain can execute young direct inputs +that the canonical scheduler still has queued at that block. The resumed frame +covers them before new user operations; the baseline itself is not a canonical +comparison checkpoint at that block. + +[Standard recovery](README.md) instead uses the existing database to repair an +optimistic suffix automatically. It assumes the local bookkeeping is trustworthy. + +## Run a rebuild + +Stop the old sequencer and resolve the cause of the failure. Choose a trusted +canonical application checkpoint; a recent one reduces replay work. Its state, +inclusion block, and next batch nonce must agree. After a bug, establish that +trust independently of the faulty local state, using a trusted canonical-machine +checkpoint or replay from a trusted origin. + +The loader requires an application artifact, `info.toml`, and `checkpoint.toml`. +The [recovery export workflow](../snapshots/lifecycle.md#http-and-recovery-exports) +describes this bundle. An export receipt checks metadata agreement; it does not +prove the checkpoint correct. An ordinary optimistic snapshot, a subscriber's +dump, or a bare local `dumps//` directory is insufficient. + +With the deployment's [setup configuration](../../README.md#running) and +batch-submitter signing key configured, use a fresh data directory: + +```sh +cargo run -p wallet-sequencer -- setup --recovery \ + --data-dir \ + --checkpoint-block \ + --checkpoint-dump-dir +``` + +Recovery signs L1 transactions, so the key must match the configured submitter. +After success, start `run` with that same data directory. A completed rebuild +refuses another `setup --recovery`; failures before completion publish no partial +baseline. + +## Implementation contract + +Read this section when changing checkpoint loading, replay, or baseline +publication. The [scheduler contract](../protocol/scheduler-semantics.md) owns +input interpretation; recovery uses that same scheduler implementation. + +### Data dictionary + +The replay boundaries are: + +| Value | Meaning | +|---|---| +| `S`, `B`, `N` | Trusted checkpoint state, inclusion block, and next batch nonce. | +| `A` | Last executed application safe block reported by `S`. | +| `C` | Fixed post-flush safe stopping block. | +| `S'`, `N'` | Recovered state and next batch nonce. | +| `K` | Application count in `S'`; the first later application input has offset `K`. | + +Loading checks the receipt's block against configured `B` and its nonce against +`info.toml`. It requires `A < B`, except for known empty genesis (`B`, nonce, and +application count all zero). At non-genesis `A = B`, a direct arriving after the +accepted batch in block `B` could still be pending but disappear from the seed +range. Checkpoint state and nonce remain operator-trusted; the later +content-identity check does not verify this prefix. + +### Flush and stopping block + +The lost database cannot supply its previous wallet-nonce watermark. Flushing +therefore depends on the provider's pool view. A transaction dropped there but +alive elsewhere may escape; a later accepted foreign or mismatched landing +freezes the rebuilt instance and requires another rebuild. The provider is +trusted fail-stop, as specified in the [threat model](../threat-model/README.md). + +After flushing, raw L1 ingestion must reach at least `C`. It may advance farther, +but the fold stops at `C`. Accepted-batch projection is deferred until the new +baseline and batch tree exist. + +### Replay boundaries + +Seed the scheduler's pending-direct queue from `(A, B]`, excluding inputs sent by +the batch submitter. Then replay **all raw inputs** in `(B, C]` in L1 order with +expected nonce `N`. Drain the remaining directs through `C` to obtain `(S', N')`. +The disjoint ranges preserve pending directs without executing the checkpoint's +accepted batches again. + +On the first `run` sync, acceptance starts at nonce `N'` and scans only blocks +**strictly after `C`**. Nonce filtering alone would let a previously rejected +future-nonce batch in the old prefix be reinterpreted as accepted. Raw inputs +ingested beyond `C` remain for normal reconciliation. Newly executed application +inputs are recorded beginning at `K`; raw L1 indices and application offsets are +separate coordinates. + +### Publish the baseline + +Write and durably sync the immutable application dump first. One SQLite +transaction then registers a fresh UUIDv4 era at generation zero, count `K`, +boundary `C`, anchor nonce `N'`, a parentless root frame at `C`, the snapshot, and +`setup_complete`. The collapsed prefix creates no application-input rows. + +Artifact failure leaves setup incomplete; transaction failure leaves at most an +orphan artifact. Identity pinning and raw L1 ingestion may survive an incomplete +attempt, but the lock and setup admission prevent serving a partial baseline. + +`C` remains the fallback reconciliation boundary if standard recovery invalidates +the root. The [history contract](../plans/application-history.md#era-baseline) +owns these immutable coordinates; [snapshot lifecycle](../snapshots/lifecycle.md) +owns restore selection, rollback-safe retention, and eventual baseline disposal. ## Code map @@ -124,4 +136,3 @@ admission exclude runtime serving before the complete baseline exists. | Atomic baseline completion | [`storage/lifecycle.rs`](../../sequencer/src/storage/lifecycle.rs) | | Scheduler fold | [`scheduler/fold.rs`](../../sequencer-core/src/scheduler/fold.rs) | | Accepted-prefix boundary | [`storage/safe_accepted_batches.rs`](../../sequencer/src/storage/safe_accepted_batches.rs) | -| Snapshot selection and GC | [`storage/snapshot_dumps.rs`](../../sequencer/src/storage/snapshot_dumps.rs) | diff --git a/docs/snapshots/README.md b/docs/snapshots/README.md index 48e8e7ed..5b983d3e 100644 --- a/docs/snapshots/README.md +++ b/docs/snapshots/README.md @@ -1,9 +1,9 @@ # Snapshots -Application snapshots are durable copies of the app's canonical state at a known -point in the L2-tx stream. They let the inclusion lane resume on startup with a -single *load-then-replay* instead of replaying all history, and they back the -operator's watchdog (`/finalized_state`) and indexers (`/latest_snapshot`). +Application snapshots are immutable, durable copies of application state at a +known execution boundary. They let the inclusion lane resume with load and +replay. A snapshot may contain optimistic state; L1 acceptance determines which +artifact can back a canonical comparison or recovery export. Two documents, split by concern: @@ -11,13 +11,12 @@ Two documents, split by concern: trait (`from_dump` / `create_dump` / `state_file_in_dump`) and the toy wallet's SSZ wire encoding. What a dump *is*. -- **[`lifecycle.md`](lifecycle.md)** — the *lifecycle* and its rationale: take at - batch close, pending → finalized promotion (per-range, atomic with the drain), - garbage collection, HTTP leasing, recovery interaction, and the crash-safety - reasoning (including the promote/drain wedge and why the design closes it). - When and how dumps move through the system, and *why*. +- **[`lifecycle.md`](lifecycle.md)** — creation at batch close, restart selection, + acceptance-derived comparison checkpoints, recovery exports, retention, + download leases, and crash safety. Acceptance is a separate durable fact; + artifacts are never promoted or rewritten. -Related: [`../recovery/README.md`](../recovery/README.md) (danger-zone recovery, -which clears cascade-doomed pendings), [`../../AGENTS.md`](../../AGENTS.md) -(architecture), and the root [`../../README.md`](../../README.md) (endpoint -shapes). +For automatic startup repair, see [standard recovery](../recovery/README.md). +For rebuilding after database loss or a sequencer bug, see +[cockroach recovery](../recovery/cockroach.md). The root +[README](../../README.md) owns endpoint shapes. diff --git a/docs/threat-model/README.md b/docs/threat-model/README.md index 077b0912..08df4af2 100644 --- a/docs/threat-model/README.md +++ b/docs/threat-model/README.md @@ -23,7 +23,7 @@ What we are protecting: | Operator env / CLI flags | Trusted, **mistakes foot-gun-guarded** | Setup configuration is authoritative — including the reviewed Uniswap V3 WETH/fee-token pool the fee oracle quotes. The complete source is pinned in deployment identity; run cannot replace it. The operator is trusted, not infallible: the supported operator-mistake class is *accidental concurrent or stale use of one data directory* — two processes on one dir (kernel process lock), a mistyped `--data-dir` (open refuses paths with no database). Deliberate operator subversion, copied-directory coordination, and distributed fencing remain out of scope (the lock is a cheap local foot-gun guard, not distributed fencing — [ADR mechanism 1](../plans/2026-08-authority-boundary-adr.md#1-runtimescope-structured-process-ownership)); mechanisms defending this class are judged against this boundary rather than re-litigated per review. | | Uniswap V3 pool (fee oracle) | Semi-trusted L1 state | Spot manipulation of a deep pool is mitigated by a 30-minute TWAP plus 10× slack in `batch_policy.log_slack`. Residual risks: TWAP lag during real moves and thin/wrong pool misconfiguration. Multi-hop pricing is out of scope. Setup writes the first Uniswap quote under the same hard L1 requirement as the rest of setup; a failed quote leaves setup incomplete. `run` does not gate recovery or admission on another quote: it starts from the persisted `batch_policy.log_gas_price`, constructs the source from setup-validated identity without RPC, and launches a refresher that logs/retries transient quote failures indefinitely while retaining that price. `log_gas_price_updated_at_ms` records successful observation for telemetry; it is not an expiry gate. A shared-endpoint outage or stale view is already caught by safe-head progress, while a pool/`observe`-specific failure with a healthy input reader is accepted as an unbounded economic residual: stale-low pricing can subsidize DA/weaken the fee spam barrier and stale-high pricing can reject users, but neither changes canonical execution because the frame's persisted fee is immutable and enforced by both sides. The 10× slack is a margin, not a proof against arbitrary market movement. Deterministic setup-time source misconfiguration (`WrongTokenPair`, `MissingPoolCode`, chain-id mismatch), fatal arithmetic, and persistent storage faults remain terminal. | | Batch-submitter private key | Private | Held in operator infra. Not reachable by the network. | -| Sequencer's own code | Trusted (bug-free is a precondition) | Bugs are prevented through tests/review and contained by fail-loud runtime invariant checks; they are not treated as adversarial behavior that the protocol can recover around. See "self-trust" below. | +| Sequencer's own code | Trusted during normal operation | Tests/review prevent bugs; runtime invariant checks fail loud. Bugs require diagnosis and correction, followed by an operator rebuild if local state cannot be trusted. See "self-trust" below. | | **L1 mempool and block builders** | **Fully adversarial** | May reorder, delay, drop, or selectively include submitted transactions. Private mempools mean "dropped" is indistinguishable from "delayed indefinitely." | | HTTP clients at `POST /tx` and `GET /fee` | Untrusted | Arbitrary public callers. May submit malformed, malicious, or replay payloads. `GET /fee` is an intentional public quote of the open-frame fee. | | WebSocket subscribers at `/ws/subscribe` | Internal, but untrusted for data-exposure | Intended for internal indexers. Treat as public for what is exposed. | @@ -31,7 +31,14 @@ What we are protecting: ### Self-trust -The sequencer trusts its own code in a specific sense: **impossible states are never *handled*.** There are no graceful fallback paths, no re-validation of a neighbor module's answer, no code that keeps running past a violated internal contract. If the sequencer emits a malformed batch, frame, or user op, it is in a bug state that requires manual intervention; normal preemptive recovery addresses liveness failures (infrastructure outages, network partitions, gateway failure), not bug-induced malformed state. Cockroach recovery is the separate operator-directed rebuild path when durable state cannot be trusted. +The sequencer trusts its own code in a specific sense: **impossible states are never *handled*.** There are no graceful fallback paths, no re-validation of a neighbor module's answer, no code that keeps running past a violated internal contract. Normal preemptive recovery addresses liveness failures such as infrastructure outages, network partitions, and gateway failure. + +If a bug corrupts local state or causes the sequencer to emit malformed batches, +frames, or user operations, the operator must diagnose and fix it. Then +[cockroach recovery](../recovery/cockroach.md) can rebuild from a trusted canonical +application checkpoint and L1. The checkpoint must be trusted independently of +the faulty local state. Historical inputs, including malformed batches, receive +the canonical scheduler's normal treatment during replay. This is **not** a prohibition on checking. Internal invariants are enforced loudly wherever a check is near-free — the type system, SQL constraints and triggers, boundary assertions — because failing loud preserves safety, while a silently-tolerated bug that externalizes (a signed batch, an ack, a feed event) is state divergence: as severe as theft and undefendable at runtime. Loud failure is not automatically self-healing: transient faults may clear on restart, but persistent invalid state is terminal and may require inspection or cockroach recovery. The rule, in short: **assert real invariants, fail loud, never absorb silently, never handle gracefully.** The decision test and the register of cross-module invariants live in [`docs/invariants.md`](../invariants.md). @@ -69,11 +76,11 @@ blocking production diagnostics would require revisiting that assumption rollbackable soft confirmations, the watchdog byte-compare, and the I15 divergence freeze. - **Adversarial mempool:** reorder, delay, drop, selective inclusion by builders -- **Zombie transactions:** a submitted batch may sit in a private mempool indefinitely and land long after we believed it was gone. Two load-bearing defenses: the recovery flusher consumes every wallet-nonce slot this deployment ever used (anchored by the persisted watermark, I14) so zombies cannot claim them; and the content-identity check (I9/I15) compares every at/above-anchor *simulated-accepted* landing against the valid closed batch we sealed at that nonce. A foreign or byte-different landing records divergence when it becomes safe and is ingested, freezes the accepted frontier, and requires cockroach recovery. This is trust-boundary validation of external input (the mempool replaying our own stale transactions at times we don't control), not defense-in-depth against self-bugs or a general canonical-state oracle. In cockroach recovery the watermark does not survive the wipe, so that flush is best-effort by construction; the content-identity check is what keeps the residual zombie detected-and-frozen rather than silent (see `docs/recovery/cockroach.md`, step 2). +- **Zombie transactions:** a submitted batch may sit in a private mempool indefinitely and land long after we believed it was gone. Two load-bearing defenses: the recovery flusher consumes every wallet-nonce slot this deployment ever used (anchored by the persisted watermark, I14) so zombies cannot claim them; and the content-identity check (I9/I15) compares every at/above-anchor *simulated-accepted* landing against the valid closed batch we sealed at that nonce. A foreign or byte-different landing records divergence when it becomes safe and is ingested, freezes the accepted frontier, and requires cockroach recovery. This is trust-boundary validation of external input (the mempool replaying our own stale transactions at times we don't control), not defense-in-depth against self-bugs or a general canonical-state oracle. In cockroach recovery the watermark does not survive the wipe, so that flush is best-effort by construction; the content-identity check is what keeps the residual zombie detected-and-frozen rather than silent (see [the flush boundary](../recovery/cockroach.md#flush-and-stopping-block)). - L1 reorgs up to safe depth - Malicious `POST /tx` callers: malformed signatures, spoofed sender, replay across chains or apps, nonce manipulation - Malicious direct-input senders: arbitrary payload, any intent; sender authenticity is guaranteed by InputBox -- Scheduler/sequencer protocol divergence of any kind (ordering, nonce rules, signature validity, fee semantics) is an in-scope correctness consequence. The content-identity check detects accepted-batch identity failures only; there is no complete runtime detector for the broader class. Shared semantics, review, and tests are preventative, and cockroach recovery is the remedy only after another signal or operator investigation diagnoses divergence. +- Scheduler/sequencer protocol divergence of any kind (ordering, nonce rules, signature validity, fee semantics) is an in-scope correctness consequence. The content-identity check detects accepted-batch identity failures only; there is no complete runtime detector for the broader class. Shared semantics, review, and tests are preventative, and cockroach recovery rebuilds state after another signal or operator investigation diagnoses divergence and its cause has been corrected. ## Out of scope From 15f799849c06624049fe277afc8adf0b945526cb Mon Sep 17 00:00:00 2001 From: gcdepaula Date: Thu, 17 Sep 2026 15:22:58 -0300 Subject: [PATCH 13/29] docs: ground automatic recovery in implementation --- AGENTS.md | 2 +- docs/invariants.md | 8 +- docs/l1-fee-policy.md | 2 +- docs/recovery/README.md | 798 ++++++++---------- docs/recovery/history/README.md | 89 +- docs/snapshots/lifecycle.md | 9 +- sequencer/src/commands/config.rs | 6 +- sequencer/src/commands/run/startup_hygiene.rs | 2 +- sequencer/src/commands/run/workers.rs | 2 +- .../src/ingress/inclusion_lane/catch_up.rs | 2 +- sequencer/src/recovery/flusher.rs | 8 +- sequencer/src/storage/ingress.rs | 5 +- sequencer/src/storage/mutations.rs | 8 +- sequencer/src/storage/recovery.rs | 193 +---- 14 files changed, 454 insertions(+), 680 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 720c22d5..ba2494d5 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -184,7 +184,7 @@ Paths below are relative to `sequencer/src/`: - **Frame** — ordering boundary; commits `safe_block` + user ops. - **Batch** — list of frames posted on-chain as one L1 transaction (SSZ-encoded). - **Inclusion lane** — the single ordering lane, with a latency-critical user-op regime and a slower L1-reconciliation regime ([ADR mechanism 4](docs/plans/2026-08-authority-boundary-adr.md)); the only writer of open batch/frame state ([I17](docs/invariants.md)) and the system's execution bottleneck. -- **Batch submitter** — stateless worker that bulk-submits all pending batches each tick. Nonces are assigned by storage (structural `parent.nonce + 1`) when batches are closed; the submitter just reads them. +- **Batch submitter** — stateless worker that bulk-submits all pending batches each tick. Storage assigns each batch's scheduler nonce at creation (`parent.nonce + 1`, or the deployment anchor for a root); the submitter reads it and selects L1 wallet nonces for submission. - **Danger detector** — polls `Storage::check_danger` and signals the process to stop so startup can recover or refuse. It reads local facts; it never writes the DB or talks to L1. - **Fee oracle** — setup pins and bootstraps a fixed price or Uniswap V3 TWAP source. The price informs future frame fees; an oracle-only outage is an accepted economic risk. The [threat model's actor table](docs/threat-model/README.md#actors-and-trust) owns the source assumptions and failure policy. - **Input reader** — ingests safe inputs from L1 InputBox and maintains the durable safe head, accepted-batch projection, and divergence marker in one atomic transaction (`sequencer/src/storage/l1_inputs.rs`); it hands the lane no in-memory cursor. diff --git a/docs/invariants.md b/docs/invariants.md index 802fe888..f9e44ac9 100644 --- a/docs/invariants.md +++ b/docs/invariants.md @@ -154,8 +154,8 @@ by writer and are write-once (`0001_schema.sql`). - **Holds:** `check_danger` checks `ClosedBatchInDanger` before `TipInDanger`. With monotonic frame clocks, the closed frontier is at least as old as the Tip. - **Enforced by:** arm order in `storage/recovery.rs` and I3. -- **Depended on by:** dispatch: a Tip-only recovery can skip flushing because - there is no doomed closed work. +- **Depended on by:** dispatch: closed-frontier danger selects flush before + a Tip-only repair can be selected. The Tip itself has no L1 footprint. ### I5. Recovery removes exactly the invalidated application suffix @@ -296,8 +296,8 @@ by writer and are write-once (`0001_schema.sql`). raises through `WalletNonceWatermarkSink` before its first send; `MempoolFlusher::flush_and_wait` likewise before its no-ops, and refuses to complete until `safe >= watermark + 1`. -- **Depended on by:** flush completeness, TLA+ Implementation Constraint 1, - cascade soundness (I9). +- **Depended on by:** [flush completeness](recovery/README.md#closed-batches-flush-sync-cascade) + and cascade soundness (I9). - **Breaks:** zombie txs evade the flush — a dropped-locally but network-surviving batch tx re-lands at a slot the recovery batch reuses, and the scheduler executes invalidated content. diff --git a/docs/l1-fee-policy.md b/docs/l1-fee-policy.md index 40ed0785..863c625f 100644 --- a/docs/l1-fee-policy.md +++ b/docs/l1-fee-policy.md @@ -38,7 +38,7 @@ expectation to measure, not a bound this policy establishes. The danger detector stops normal operation when the configured danger threshold is reached. That bounds continued soft-confirmation issuance under the detector's assumptions; it does not bound inclusion or recovery duration. -[Recovery](recovery/README.md#step-4-post-flush-state) requires all covered +[Recovery](recovery/README.md#closed-batches-flush-sync-cascade) requires all covered wallet slots to resolve at safe depth before a cascade can proceed. The flusher's fixed headroom can also fail to replace an unmineable original. The sequencer remains offline until recovery succeeds or the operator acts. diff --git a/docs/recovery/README.md b/docs/recovery/README.md index 6513d28e..a1bdf1e2 100644 --- a/docs/recovery/README.md +++ b/docs/recovery/README.md @@ -1,491 +1,353 @@ -# Batch Recovery - -There are two recovery paths: - -- **Standard recovery**, described on this page, runs automatically at startup. - It uses the existing database to repair an optimistic suffix that can no - longer be relied on and decides whether the sequencer can resume serving. -- **[Cockroach recovery (`setup --recovery`)](cockroach.md)** is an operator-triggered - rebuild from a trusted canonical application checkpoint and L1. Use it after - database loss or unusable local state, including after fixing a sequencer bug. - -Two complementary bounded TLA+ models cover standard recovery: -[`preemptive.tla`](preemptive.tla) for batch/slot safety and -[`admission.tla`](admission.tla) for startup phase ordering and admission. They -do not currently model the external era/generation/base metadata, the canonical -`application_inputs` projection, or snapshot artifact/GC lifecycle; their crash -atomicity is enforced by SQLite transaction boundaries and schema triggers. - -## Runtime lifecycle at a glance - -The sequencer's recovery loop spans two process lifetimes: - -1. **In-process detection.** The `DangerDetector` polls `Storage::check_danger`. Expected-recovery and retryable exits close intake and drain workers before returning non-zero. A terminal fault aborts the process immediately ([ADR mechanism 1](../plans/2026-08-authority-boundary-adr.md#1-runtimescope-structured-process-ownership)). -2. **External respawn.** An orchestrator (systemd, k8s, …) restarts expected-recovery and retryable exits. A terminal exit (30 or SIGABRT) requires operator investigation before a deliberate restart. -3. **Startup recovery.** Under the process lock, before workers exist, startup checks local terminal facts, attempts an initial L1 Sync, then selects at most one repair from a consistent `RecoveryInspection`: open a missing Tip, replace an aging Tip, or Flush → Sync → Cascade. Repair is followed by a fresh check. -4. **Prepare, admit, launch.** A clean result permits task-free, fallible preparation. A final current inspection must still be clean to mint the single-use `RuntimeAdmission` witness; worker launch consumes it synchronously. - -The detector trip and the startup dispatch share the same `check_danger` function; the detector cares only that *some* arm fired, while the startup dispatch examines *which* arm fired to pick the right action. - -Key abstractions, by responsibility: - -- **`DangerDetector`** ([`recovery/detector.rs`](../../sequencer/src/recovery/detector.rs)): reads danger on a cadence and exits on any non-`Safe` status. It writes nothing and performs no L1 calls. -- **`BatchSubmitter`** ([`l1/submitter/worker.rs`](../../sequencer/src/l1/submitter/worker.rs)): makes L1 progress; the detector owns danger checks. -- **Startup recovery** ([`recovery/mod.rs`](../../sequencer/src/recovery/mod.rs)): a sequential procedure with one exhaustive dispatch shared by repair selection and final admission. Error classification belongs here; command settlement consumes the resulting retry/refuse verdict. -- **Guarded recovery storage** ([`storage/recovery.rs`](../../sequencer/src/storage/recovery.rs)): checks each repair's preconditions and commits its mutation atomically. The cascade checks divergence and the flush-view floor before changing the batch tree. -- **`MempoolFlusher`** ([`recovery/flusher.rs`](../../sequencer/src/recovery/flusher.rs)): consumes unresolved wallet-nonce slots and waits for safe finality. Provider errors leave the attempt; the orchestrator retries. -- **`ProtocolTiming`** ([`sequencer-core/src/protocol.rs`](../../sequencer-core/src/protocol.rs)): shared scheduler timing plus sequencer-local danger and clock policy. - -Procedure tests use the real SQLite inspections and repair transactions, substituting only Sync and Flush at the L1 boundary. - -## The Batch Tree - -Batches form a tree where each node is a batch and edges point from child to parent. Each batch has a single parent: the preceding batch in the valid chain. - -Batches have two identifiers: - -- **Index** (`batch_index`): monotonically increasing, unique, never reused. Creation order. -- **Nonce** (`batch_nonce`): scheduler sequence number. Storage derives it as - `parent.nonce + 1`, or the deployment's anchor nonce for a parentless root. - -In normal operation the tree degenerates into a list -- index and nonce increase in lockstep. Branches appear only after recovery, when a suffix of the chain is invalidated and a new batch forks from the last valid ancestor. - -There is always exactly one **valid path** (root to leaf) that constitutes the current batch chain. The valid path splits into a **prefix** (safe on L1, accepted by the scheduler) and a **suffix** (pending or confirming). - -### Genesis sentinel (nonce-0 edge case) - -Recovery requires at least one Gold ancestor (the cascade invalidates a suffix and forks from the last Gold batch). If the very first batch (nonce 0) goes stale before any batch becomes Gold, there is no ancestor to fork from. - -The TLA+ model handles this with a **genesis sentinel**: the initial state starts with a Gold batch at nonce 0. This is a modeling technique that eliminates the nonce-0 special case, allowing Resolve to use uniform logic (the `fng > 1` guard is always satisfied). Without it, the model would need a separate Resolve action with different arithmetic for the "no Gold ancestor" case. - -The implementation handles the nonce-0 case **structurally**: `open_fresh_tip_in_tx` (`storage/ingress.rs`) roots a nonce-0 batch whenever the valid path is empty (genesis, or a fully-torn cascade) — no sentinel batch is submitted and no recovery branch is special-cased. The model's sentinel and the implementation's structural root play the same role. - -#### Cockroach recovery generalizes the root nonce (the anchor) - -A fresh deployment roots its batch tree at nonce 0. Cockroach recovery roots it -at the scheduler's next nonce after replay. The `batch_tree_anchor` preserves -that starting nonce even if a later cascade removes the whole local branch. -[I16](../invariants.md#i16-the-batch-tree-has-exactly-one-valid-parentless-root-carrying-the-deployments-anchor-nonce) -owns the root invariant and its enforcement. - -## Coloring - -Every batch on the valid path has exactly one color. Dead branches are lead (permanently invalid). - -### Simplified model (three colors) - -| Color | Meaning | Terminal? | -|------------|----------------------------------------------------------------|-----------| -| **Gold** | Safe on L1 and accepted by the scheduler | Yes | -| **Silver** | Valid, optimistically executed, but not yet safe/accepted | No | -| **Lead** | Invalid (has `batches.invalidated_at_ms` set) | Yes | - -Gold batches form a contiguous prefix of the valid path. Silver batches form a contiguous suffix (after the gold prefix up to the open batch). Lead batches hang off gold nodes as dead branches -- the first lead in any cascade always has a gold parent. - -### Extended model (five colors) - -To model the full lifecycle including L1 submission: - -| Color | Meaning | Has `w_nonce`? | -|-------------|--------------------------------------------------------|----------------| -| **Tip** | Open batch, not yet closed | No | -| **Pending** | Closed, may or may not be submitted to mempool | Maybe | -| **Bronze** | Included in an L1 block, block not yet safe | Yes | -| **Silver** | Included, block has reached safe finality | Yes | -| **Gold** | Safe, accepted and executed by the scheduler | Yes | - -The spine ordering invariant: `Gold* Silver* Bronze* Pending* Tip` - -A Pending batch may have a `w_nonce` (submitted to the L1 mempool but not yet included in a block) or not (not yet submitted). The batch submitter assigns `w_nonce`s to all unsubmitted Pending batches at once, in spine-position order. - -## Nonce Poisoning - -The scheduler maintains a single counter: "I expect batch nonce N next." - -When a batch with nonce N arrives stale, the scheduler **skips it entirely** -- no nonce increment, no state change, no report. It is a true noop in nonce-space. - -This poisons the nonce counter. Every subsequent batch (nonce N+1, N+2, ...) is dead on arrival. Not because they are individually stale, but because the scheduler still expects nonce N. The only batch with nonce N was stale and skipped, so the counter will never advance past N. - -Cascade invalidation is therefore **exact, not conservative**. The sequencer's `WHERE batch_index >= stale_batch_index` mirrors precisely what the scheduler will do (refuse). The entire silver suffix is unreachable once any batch in it is stale. - -Recovery is the only way forward: create a new batch with nonce N, giving the scheduler what it needs to resume. - -## Two Staleness References - -The staleness formula is `reference_block - first_frame_safe_block >= MAX_WAIT_BLOCKS`, but the reference block differs by context: - -### Inclusion staleness (scheduler's perspective) - +# Automatic Recovery + +Automatic recovery repairs the sequencer's optimistic history after a liveness +failure. It stops issuing soft confirmations, settles outstanding submissions +when necessary, and replaces the affected suffix before resuming. It uses the +existing SQLite database and assumes the sequencer's own code and accepted +local history are correct. + +For lost or unusable local state, including after a sequencer bug, use +[cockroach recovery](cockroach.md): fix the bug, choose a trusted canonical +application checkpoint, and rebuild from L1 in a fresh data directory. Automatic +recovery does not establish trust in a corrupted application state. + +This page owns the automatic procedure and its rationale. Start with the +lifecycle and dispatch below; read the safety arguments and model boundaries +when changing recovery. The [scheduler contract](../protocol/scheduler-semantics.md) +owns canonical acceptance rules; the [invariant register](../invariants.md) owns +cross-module enforcement. + +## The state being repaired + +The local batch tree has one valid path: an **accepted prefix**, followed by an +**optimistic suffix** ending at the open **Tip**. Recovery invalidates a suffix +and opens a new Tip from the surviving path. Invalidated batch, frame, and +user-op source facts remain available for audit. + +“Accepted” (also called **Gold** in code and models) means the safe-input +projection applied the scheduler's acceptance rules and matched the landed +bytes to a valid local closed batch. It does not mean an independent canonical +machine was observed executing it. A foreign or different accepted payload +records canonical divergence and forbids automatic repair. + +Keep three identities separate: + +| Identity | Meaning during recovery | +|---|---| +| Local `batch_index` | Unique creation identity; never reused. Invalidation targets a local suffix. | +| Scheduler batch nonce | Ordering identity; derived when storage creates the batch. A replacement branch reuses the invalidated suffix's nonces. | +| L1 wallet nonce | Transaction slot; a batch transaction and a flush no-op may compete for it. Covered slots must be consumed at safe depth before post-flush repair. | + +A parentless root uses the deployment's immutable anchor nonce: zero at genesis, +or the scheduler's next nonce after cockroach recovery. Production needs no +accepted ancestor or submitted sentinel to repair a fully invalidated branch +([I16](../invariants.md#i16-the-batch-tree-has-exactly-one-valid-parentless-root-carrying-the-deployments-anchor-nonce)). + +## Lifecycle and startup dispatch + +The recovery cycle crosses a process boundary: + +1. **Detect and stop.** `DangerDetector` polls local `Storage::check_danger`; + any non-`Safe` result stops normal operation. It neither writes the database + nor calls L1. Expected-recovery and retryable exits close intake and drain + workers; diagnosed terminal runtime faults abort immediately. +2. **Respawn.** The orchestrator restarts expected recovery (`10`) and retryable + exits (`20`). Terminal exit `30` or `SIGABRT` requires investigation before a + deliberate restart. Process + ownership and shutdown belong to the [authority-boundary ADR](../plans/2026-08-authority-boundary-adr.md). +3. **Recover under the process lock, with no workers.** Check local terminal + facts before any provider call, attempt initial L1 sync, select at most one + repair, then inspect again after a repair. +4. **Prepare, admit, launch.** Prepare resources without starting tasks. A final + current inspection must still authorize serving. It creates a single-use + `RuntimeAdmission` witness, consumed synchronously by worker launch. + +Startup first refuses a persisted canonical divergence or a missing rollback-safe +checkpoint. After initial sync, one consistent `RecoveryInspection` selects: + +| Local fact | Action | +|---|---| +| `Safe` + open Tip | Ready for runtime preparation. | +| `Safe` + no Tip | `EnsureOpenTip`: create it under a transaction guard. | +| `TipInDanger(N)` | `RecoverTip { N }`: invalidate that Tip and reopen, without flushing. | +| `ClosedBatchInDanger(N)` | Flush → sync → guarded post-flush cascade. | +| `L1ViewStale` | Retry; the persisted view or clock cannot authorize serving. | +| `EstimatedBatchInDanger(N)` | Retry; an estimate alone cannot authorize invalidation. | +| `CanonicalDivergence(N)` or missing recovery checkpoint | Refuse; automatic recovery cannot repair this trust failure. | + +Only an initial-sync **provider failure** may fall back to a still-usable +persisted view. Other failures keep their typed retry/refuse classification. +Post-flush sync has no such fallback: it must establish the view required by +the cascade. + +Every repair must commit an open Tip. A fresh inspection afterward must report +`Safe` with a Tip and no terminal facts; otherwise the boot exits. Startup does +not attempt a second repair in that invocation. Final admission repeats the +same policy after preparation, because preparation can outlive the freshness +of the L1 view. A new repair requirement also exits rather than launching. + +Preparation validates the rollback checkpoint's metadata and performs snapshot +hygiene, but does **not** restore the application. After launch, the inclusion +lane loads a surviving snapshot and completes catch-up before processing new user ops. +Admission authorizes worker launch; application restoration can still fail. + +Startup logs `danger_status`, `danger_batch_index`, and `recovery_decision`, +then any invalidated indexes. Errors retain their retry/refuse classification +and diagnostic cause; the orchestrator owns restart policy and alert routing. + +## Detection and timing + +Canonical staleness and local danger use different reference blocks and thresholds: + +```text +scheduler rejects a batch with at least one frame when: + inclusion_block - first_frame.safe_block >= MAX_WAIT_BLOCKS + +sequencer observes danger when: + current_safe_block - first_frame.safe_block >= danger_threshold + danger_threshold = MAX_WAIT_BLOCKS - preemptive_margin_blocks ``` -inclusion_block - first_frame_safe_block >= MAX_WAIT_BLOCKS -``` - -Used by `populate_safe_accepted_batches` to simulate what the scheduler accepts. Each batch has its own inclusion block (the L1 block where its submission landed). **Not monotonic** across batches -- a promptly submitted old batch can be healthy while a late-submitted newer batch is stale. - -Inclusion staleness determines the **gold frontier**: the set of batches the scheduler has accepted. -### Current staleness (sequencer's detection) - -``` -current_safe_block - first_frame_safe_block >= MAX_WAIT_BLOCKS +An old batch can already have landed while fresh. Its inclusion block decides +acceptance; its age at the current safe head does not undo that acceptance. +The detector therefore examines the first unaccepted closed batch and the Tip, +not accepted history. A wire batch with zero frames is never stale and consumes +its nonce; a normal local batch with zero user ops still has a first frame and +can age. + +### Danger threshold + +The threshold means “stop and recover,” not “this batch cannot land.” The Tip +may still be canonically fresh when invalidated, and closed batches can become +accepted while the flush is running. + +The margin provides headroom before canonical expiry; it is not a grace period +after detection. Startup can repair immediately. Defaults and validation live +in [`TimingArgs`](../../sequencer/src/commands/config.rs): with `MAX_WAIT_BLOCKS` +1200 and margin 300, observed danger starts at age 900 blocks. Neither that +margin nor the fee policy bounds the time required to finish recovery. + +### When safe-head progress stops + +A responsive RPC endpoint can keep returning an old view. The detector uses +both the safe block's timestamp and local time since the last recorded safe-head +advance. [`check_danger_in`](../../sequencer/src/storage/recovery.rs) checks in +this order: + +1. Canonical divergence. +2. Missing or old safe-block timestamp → `L1ViewStale`. +3. Observed closed-batch danger, then observed Tip danger. +4. Clock regression of at least one block-time against either persisted time + baseline → `L1ViewStale`; sub-block skew is tolerated. +5. Estimated missed blocks (`elapsed / seconds_per_block`) reduce the danger + threshold. An unresolved batch crossing it gives `EstimatedBatchInDanger`. +6. Otherwise `Safe`. + +An old view blocks repair selection before observed-age checks. A regressed +clock does not suppress danger already established by observed block numbers; +a remaining clock fault still prevents admission after repair. Estimates stop +new soft confirmations but never decide which work to invalidate. + +## Repairs and their guards + +### Closed batches: flush, sync, cascade + +**Flush the covered wallet slots.** The durable wallet-nonce watermark `W` is an +upper bound on every slot this deployment may have broadcast. Every broadcaster +raises it durably **before** sending at a new nonce. A crash between those steps +may cover a slot that was never used; it must not leave a sent slot uncovered +([I14](../invariants.md#i14-watermark--wallet-nonce-of-every-tx-ever-broadcast)). + +The flusher submits zero-value self-transfers at unresolved slots from the +account's Latest nonce through `max(Pending, W + 1) - 1`. It completes only when: + +```text +Pending <= Safe && Safe >= W + 1 ``` -Used by the danger threshold detector. The reference block (`current_safe_block`) is the same for all batches. **Monotonic within the valid path** -- earlier batches have smaller `first_frame_safe_block`, so larger difference. If the frontier batch is not stale by this measure, no batch is. - -Current staleness triggers **preemptive recovery** (see below). - -## Nonce Uniqueness on the Valid Path - -`batches.nonce` can repeat across the full table -- a recovery batch inherits `parent.nonce + 1` from the last valid ancestor, which is the same nonce the first invalidated suffix batch had. Among **valid batches** (those with `invalidated_at_ms IS NULL`), nonces are unique because the valid path is a strict chain via `parent_batch_index`. - -This matters because L1 works in nonce-space (the scheduler identifies batches by nonce) while the sequencer works in index-space (local `batch_index`). The recovery path needs to translate between them: "which batch indexes should we invalidate?" Nonce uniqueness on the valid path is what makes this mapping unambiguous. - -## The L1 Stream - -L1 processes transactions in `w_nonce` order. At each slot (a given `w_nonce` value), exactly one transaction is included. If multiple transactions compete for the same slot (e.g., a dead batch and a flush no-op), L1 non-deterministically picks one. The loser is discarded. - -This is the interface between the sequencer and the scheduler. The scheduler sees a stream of entries ordered by `w_nonce`, each with a `batch_nonce`, `inclusion_block`, and `safe_block`. It processes them in order, accepting or rejecting based on nonce match and staleness. - -## The Uncertainty Interval - -The core insight behind the recovery design is that **mempool uncertainty is bounded by a time interval**. - -Once a batch's `safe_block` is old enough that `current_safe_block - safe_block >= MAX_WAIT_BLOCKS`, we know it is stale no matter when it lands on L1 (because `inclusion_block >= current_safe_block`). Any batch in the mempool with that `safe_block` is dead-on-arrival. This means mempool uncertainty has a natural expiration: after `MAX_WAIT_BLOCKS`, the L1 outcome doesn't matter. - -This gives us three regimes: - -``` -|---------- safe ----------|-- danger zone --|-- past MAX_WAIT --| - no action flush + recover self-resolved -``` - -- **Before the danger zone**: batches are young. Nothing to do. -- **In the danger zone**: batches might land stale, or might still make it. This is the window of uncertainty. For **closed unresolved batches**, the flush resolves it by forcing every `w_nonce` slot to finalize (batch wins or no-op wins). After the flush, the sequencer reads the scheduler's finalized state and cascades if needed. An **open Tip** has no `w_nonce` slot yet, so it is not part of this uncertainty set. -- **Past MAX_WAIT**: all unresolved batches are guaranteed stale by L1 monotonicity (`inclusion_block >= current_safe_block >= safe_block + MAX_WAIT`). For closed unresolved batches, the L1 outcome no longer matters because every eventual inclusion is stale, but wallet-nonce slots may still need to be flushed (or naturally consumed) before recovery can reconstruct the scheduler frontier. For an aging open Tip, there is no L1-slot uncertainty at all, so startup recovery can invalidate it directly. - -**What TLA+ proves vs external reasoning**: the TLA+ model ([`preemptive.tla`](preemptive.tla)) proves that after all `w_nonce` slots are resolved (however that happens), ZombieSafety holds. It does not model the danger threshold or the passage of time. The claim that "past MAX_WAIT, staleness self-resolves" is an external argument from L1 monotonicity (`inclusion_block >= current_safe_block`), not something TLA+ checks. - -Any recovery design must wait out this uncertainty. The question is how. The preemptive design (implemented here) forces resolution by going offline and flushing. An alternative optimistic design lets the uncertainty resolve naturally but keeps serving soft confirmations -- see [`history/`](history/) for that approach and why we preferred preemptive. - -## Silver-Only for Submitted Batches - -The Silver-only constraint applies to **submitted batches whose L1 slot outcome is still relevant**. This is the zombie path, and it is where the optimistic-design counterexample from [`history/`](history/) still matters. - -A Silver batch's L1 entry is permanent -- no mempool competition can kill it. The scheduler **will** see it, at a `w_nonce` lower than any recovery batch, and be poisoned. This ordering guarantee is what makes nonce poisoning reliable. - -Detecting staleness on Pending or Bronze submitted batches *before wallet-nonce uncertainty is resolved* is unsafe: a recovery batch can take the frontier's L1 slot via wallet-nonce mutual exclusion, preventing the scheduler from ever seeing the stale frontier, and allowing non-frontier dead batches to pass the nonce check. TLA+ model checking found this bug; see [`history/`](history/) for the counterexample. - -The open Tip is different. It has no L1 transaction yet, so there is no `w_nonce` competition and no zombie risk. Once `current_safe_block - first_frame_safe_block >= danger_threshold`, startup recovery can invalidate the aging Tip directly and open a fresh one. Likewise, after a preemptive flush has resolved all competing `w_nonce` slots for closed batches, the atomic recovery transaction can safely use **current staleness** on the oldest unresolved batch (closed or open). - -## Preemptive Recovery Design - -The sequencer uses a preemptive approach: detect danger early, go offline, flush the mempool, then recover on solid ground. This design was preferred over the optimistic alternative because it is simpler to reason about and produces fewer invalidated soft confirmations (the sequencer stops issuing them before the cascade). - -### Step 1: Danger threshold - -Define `DANGER_THRESHOLD = MAX_WAIT_BLOCKS - MARGIN`. When the frontier batch's current staleness (`current_safe_block - safe_block`) reaches `DANGER_THRESHOLD`, **trigger preemptive recovery**. - -The threshold is *only* a trigger. It says "stop running, hand off to recovery." It does **not** say "this batch is doomed." The cascade decision belongs to step 5, which examines the post-flush state and acts on what's actually there. - -#### Why a margin at all (Sorites argument) - -The right value of `MARGIN` is not derived from the recovery procedure's runtime — it falls out of a sharper question: **at what age do we give up on the current batches and start anew?** - -Two endpoints are clear: - -- A batch that's 1 minute behind shouldn't be invalidated. The infra hiccup might pass; pre-confirmations issued against it will likely still land. -- A batch that's 1 minute *before* `MAX_WAIT_BLOCKS` shouldn't be left to die. We've already tried for hours. The last minute won't save us, and pre-confirmations issued in this window are knowingly dishonest — we have strong evidence they won't land. - -Somewhere between those, we want to switch from "keep waiting" to "give up." The exact crossover is a Sorites question with no canonical answer, but two design pressures pin it: - -1. **Stop issuing pre-confirmations on state we reasonably know won't land.** As current staleness approaches `MAX_WAIT_BLOCKS`, the probability that the current batch lands gracefully drops. Pre-confs issued past that point are increasingly dishonest to users. -2. **Give the operator runway to fix infra.** If L1 is misbehaving, network is degraded, mempool is congested — the operator needs hours, not minutes, to diagnose and act before the system commits to recovery and invalidates work. - -The recovery procedure's own runtime (flush submission + L1 safe finality wait of ~13 min on Ethereum + atomic SQLite cascade) is a *floor* on `MARGIN`, not the deciding factor. It must fit, but fitting it is far from the operating point. - -#### Defaults - -With `MAX_WAIT_BLOCKS = 1200` (~4 hours), the default `MARGIN = 300` blocks (~1 hour at 12s/block) gives the operator ~1 hour after danger-zone entry before the system commits to recovery. That's well above the procedure-runtime floor (~15 min) and meaningful runway under the second design pressure. - -Production tunings with a longer `MAX_WAIT_BLOCKS` (e.g. 24h) should keep the margin in the hours range — there's no benefit to a tighter margin once `MARGIN` exceeds the procedure-runtime floor several times over. - -### Step 2: Go offline - -Stop accepting new user operations. From the outside world, the sequencer is temporarily unavailable. This eliminates concurrent batch creation during recovery. - -### Step 3: Flush mempool - -Read the persisted **wallet-nonce watermark** `W` — the highest `w_nonce` this deployment ever broadcast (`wallet_nonce_watermark` singleton; see Implementation Constraint 1). Query the latest confirmed `w_nonce` (N) and the pending `w_nonce` (M). Submit no-op transactions (self-transfers of 0 ETH) at nonces N, N+1, ..., `max(M, W+1) - 1`. These compete with any of our transactions still alive anywhere in the network — including zombies the local node's pool has forgotten. - -Wait until both `pending <= safe` **and** `safe >= W + 1`: every slot this deployment ever used is consumed at safe depth. The second conjunct is the durable anchor — without it the flush trusts the local node's volatile mempool memory, which a dropped-locally-but-alive-elsewhere zombie evades entirely. The flush reports the safe block at which it observed resolution; Step 5 refuses to cascade until the re-synced view reaches at least that block. - -### Step 4: Post-flush state - -Every `w_nonce` slot from N to M-1 is now resolved: - -- **Batch won**: the batch is on L1 and safe (Silver or Gold) -- **No-op won**: the batch is dead forever, its slot consumed - -There are no more mempool entries. All uncertainty is resolved. - -**Flush safety does not depend on eviction; completion depends on L1 progress.** -A rejected no-op surfaces as a hard `FlushError` and the process exits. The -orchestrator respawn re-runs the flush. Inclusion of either the original batch -or a no-op can resolve the slot, but the sequencer remains offline until every -covered slot reaches safe depth. Neither retries nor the danger threshold -establish a recovery deadline. - -No-ops use 3× the fresh fee estimate, followed by a symmetric replacement bump. -This headroom improves their chance of replacing an earlier transaction; it -does not guarantee replacement. Base fees and priority estimates can move in -opposite directions. For example, a poster tx sent at base 10 gwei with cap -22 gwei and tip 2 gwei cannot mine at base 30 gwei. If the current tip estimate -is 0.5 gwei, the no-op offers cap 199.65 gwei and tip 1.65 gwei (plus 1 wei on -each). Geth rejects that replacement because its tip misses the 2.2 gwei -threshold. Both the original and the no-op can therefore fail to make progress. -A previous flush no-op can also block another pass on a flat market. These are -accepted liveness limits of the current [fee policy](../l1-fee-policy.md), not -permission to cascade before the slots resolve. - -### Step 5: Run recovery - -This is an atomic SQLite transaction operating on the best available L1 state. The storage work splits cleanly by whether a flush ran first. - -#### Mental model: "everything past gold is doomed" - -After the flush has resolved every wallet-nonce slot, and `populate_safe_accepted_batches` has been re-synced, the gold spine is at its **maximum extent**: the simulation walked safe-inputs in inclusion order, accepting each one until it hit a barrier (a stale batch, or a missing batch where a no-op consumed the slot). - -Any batch past that gold frontier is **doomed**, in one of three concrete senses: - -| State | What happened | Why doomed | -|---|---|---| -| **Silver-stale** | Original tx landed, scheduler skipped (`inclusion_block - first_frame ≥ MAX_WAIT`) | Scheduler's expected nonce never advances past it; downstream batches are nonce-poisoned | -| **Silver-fresh poisoned** | Original tx landed fresh, but a preceding stale or missing batch poisoned the nonce | Scheduler skipped on nonce mismatch; on-chain row can't be retroactively re-evaluated | -| **Pending (no-op'd)** | Flush no-op consumed the wallet-nonce slot; original tx never landed | The L1 transaction is dead. Re-submission at a fresh slot would land *after* the existing on-chain Silver-poisoned batches; the scheduler sees those at lower `safe_input_index`, advances expected past them on the resub generation, but the per-original-tx work is gone | - -**Why isn't this just "stale"?** Under self-trust (we don't defend against malformed self-submissions), the *first* non-gold closed batch can only be Silver-stale or Pending. Nonce-mismatch is impossible at the frontier — nonces are contiguous on the valid path (`trg_enforce_nonce_contiguity`). But *downstream* batches past that first non-gold are typically Silver-fresh-poisoned: their inclusion-staleness was fine, but they were processed when expected was stuck at the poisoned nonce. - -A **fourth shape** sits outside this taxonomy: a closed batch that was **never submitted** (closed after the submitter's last tick before the detector exit). It has no L1 footprint, no killed tx, and is not literally doomed — it could simply be submitted after recovery. The cascade invalidates it anyway: once committed to recovery, cascading the entire non-gold suffix converges in one cycle and avoids spine-order reasoning about a half-submitted suffix. The cost is real (its soft confirmations are rolled back); this is a deliberate convergence-over-preservation policy choice. - -Cascading from the first non-gold catches all four. **No per-batch age check is needed for the cascade pivot itself** — every closed batch past gold is either doomed by construction or sacrificed by the convergence policy. - -#### Path A — guarded post-flush Cascade - -After step 3 (flush) and step 4 (re-sync), the gold frontier is fresh. Run the atomic recovery transaction: - -1. **Find the cascade pivot.** First try the closed pivot: first valid closed batch with `nonce >= frontier_nonce`. By the contiguity invariant, this batch's nonce is exactly `frontier_nonce`. If one exists, cascade from it. -2. **No closed pivot? Check the Tip.** When all closed batches landed fresh and were accepted (the "everything worked" aftermath), there's no closed pivot — but the Tip can still be in the danger zone. When the lane rotates without a safe-block advance between frames (e.g. immediately after init, both frames share the bootstrap `safe_block`), `S_tip = S_closed`. The closed batch can become gold by inclusion-staleness while the Tip's age — measured against `current_safe_block` after the flush wait — has crossed the danger zone. Pure monotonicity (`S_tip ≥ S_closed`) doesn't rule this out: equality is allowed. So fall through to `find_tip_batch_in_danger(danger_threshold)`. If the Tip's age clears `danger_threshold`, cascade it. -3. **Cascade-invalidate the suffix**: set `invalidated_at_ms` on every valid batch with `batch_index >= pivot.batch_index`. This catches all non-gold batches in cases (2)/(3) above, and the Tip alone in the no-pivot-but-Tip-aging case. The invalidation trigger deletes those batches' canonical `application_inputs` rows, rewinding head `H` to the surviving prefix. Raw L1, batch, frame, and user-op source facts remain available for audit. -4. **Advance external history reality**: iff step 3 invalidated at least one valid batch, increment `RecoveryGeneration` exactly once in this same SQLite transaction. A no-invalidation repair does not bump it. Application-history rewind and generation change are therefore one visible transition. -5. **Open recovery batch**: parent is the last valid ancestor (`MAX(batch_index) FROM valid_batches` after the cascade). Nonce is structurally `parent.nonce + 1`, which equals `frontier_nonce` — the scheduler's `expected_nonce`. Reconcile external directs after the latest surviving frame's `safe_block`, or immutable baseline block `C` if no frame survives. The new application rows reuse offsets beginning at the rewound head under the incremented generation. - -**Threshold = `danger_threshold`, not `MAX_WAIT_BLOCKS`**. We're already committed to recovery; the Tip is past gold; if it's also past the threshold that would have triggered recovery had it been a closed batch, cascade it. Otherwise the next danger detector tick after resume would re-trip on the Tip's eventual close + submission anyway (the closed batch would inherit its first frame's safe_block). - -#### Path B — guarded `RecoverTip` - -The `RecoverTip` action is dispatched when `check_danger` returns `TipInDanger(idx)`: no closed batch is past the gold frontier in the danger zone, but the open Tip's first frame has aged past `danger_threshold`. **No flush ran** — the Tip has no L1 footprint, so there's nothing to flush. - -Closed batches past gold (if any) are still in their natural lifecycle — pending in the mempool, recently included, awaiting safe finality. Cascading them would prematurely abort their progression. We act only on the Tip: - -1. Run `find_tip_batch_in_danger(danger_threshold)`. If `Some(tip_index)`, cascade-invalidate from there (which only touches the Tip — no closed batches have `batch_index >= tip_index`) and increment `RecoveryGeneration` exactly once in that same transaction. -2. Open a fresh recovery batch in the same transaction. -3. If no Tip in danger and no Tip exists at all (torn-state crash recovery), open a Tip anyway. - -The `Safe` decision with no open Tip runs `EnsureOpenTip`. Its transaction rechecks `Safe`, rollback-safe checkpoint presence, and Tip absence, then opens the Tip through `open_fresh_tip_in_tx`. It refuses rather than commit without an open Tip. Startup rechecks danger after this repair; Tip creation never occurs as a worker-construction side effect. - -#### Why `danger_threshold`, not `MAX_WAIT_BLOCKS`, for the Tip threshold - -The Tip threshold is a **policy choice**, not a mathematical staleness bound. A Tip whose first frame is at age `danger_threshold` could in principle still close, submit, and land fresh by inclusion-staleness — `inclusion_block - first_frame` would be roughly `danger_threshold + (rotation + submit latency)`, which (with a reasonable margin) is still below `MAX_WAIT_BLOCKS`. - -We invalidate at `danger_threshold` because: - -1. **Pre-confirmation honesty.** Once the Tip's age crosses the danger zone, the system has decided this generation is operationally suspect. Continuing to issue soft confirmations against it is dishonest to users. -2. **Avoid retrip risk.** The runtime danger detector also fires on `DangerStatus::TipInDanger`. Without invalidating at startup, we'd resume operation, the detector would re-trip on the next tick, and we'd cycle. Cascading at startup converges in one cycle. -3. **Symmetry with the closed-batch trigger.** The closed-batch detector trips at `danger_threshold`. Using the same threshold for the Tip preserves the framing: "danger zone = committed to recovery." - -### Step 6: Resume - -Restart the batch submitter and user-op acceptance. If this recovery invalidated -any valid batch, the generation bump already committed atomically with that -invalidation; otherwise the history version is unchanged. The sequencer is -back online. - -### Why post-flush cascade is unconditional (and not threshold-based) - -An earlier design considered using `MAX_WAIT_BLOCKS` as the cascade threshold even in the post-flush path: only invalidate the frontier if its `current_safe_block - first_frame.safe_block ≥ MAX_WAIT`. The intuition was to preserve soft confirmations when re-submission could still land fresh. - -**This doesn't hold up.** Walk through the boundary case: - -1. Frontier batch has `current_staleness ∈ [danger_threshold, MAX_WAIT)`. Detector trips, flush runs. -2. `recover_post_flush` (with hypothetical threshold) sees age below MAX_WAIT, declines to cascade. Resume. -3. Submitter wakes up, resubmits the Pending frontier (and any non-gold closed batches) at fresh wallet-nonce slots. They enter the mempool. -4. Detector polls again. Frontier age has barely moved or the published safe - head is unchanged; providers may later expose several newly-safe blocks as - one jump, but no cadence assumption makes the frontier clean again. It is - still above `danger_threshold`, so the detector trips again. -5. Recovery 2 starts. Flush submits no-ops at the slots the submitter just used for resubs. Bumped fees on no-ops typically out-bid resubs. Resubs killed. -6. Goto step 2. Loop converges only when `current_staleness` finally crosses `MAX_WAIT_BLOCKS` and the threshold check fires. - -Each loop iteration burns gas (no-ops + doomed resubs), takes ~12 minutes (the flush's safe-finality wait), and the soft confirmations are rolled back at the end anyway. Cascading on first non-gold converges in **one cycle** with predictable cost. - -### Startup behavior summary - -Startup holds the exclusive process lock and launches no workers until recovery and preparation finish. Its first local inspection refuses canonical divergence or a missing rollback-safe checkpoint before any provider call. It then attempts one initial Sync: a provider failure may use a still-fresh persisted view, while other failures retain their typed retry/refuse classification. - -After that attempt, `select_recovery` maps one consistent local inspection as follows: - -| Local fact | Action | Why | -|---|---|---| -| `Safe` + open Tip | Ready for preparation | The local prediction is clean and structurally resumable. | -| `Safe` + no Tip | `EnsureOpenTip` | Open the Tip under its transaction guard, then recheck. | -| `L1ViewStale` | Retry | The persisted view cannot authorize new soft confirmations. | -| `TipInDanger(N)` | `RecoverTip { N }` | The Tip has no L1 footprint; invalidate and reopen directly. | -| `ClosedBatchInDanger(N)` | Flush → Sync → Cascade | Resolve the closed batches' L1 slots before changing their local suffix. | -| `EstimatedBatchInDanger(N)` | Retry | Recovery never mutates from an estimate alone. | -| `CanonicalDivergence(N)` | Refuse | Standard recovery assumes content identity and is forbidden. | - -Closed recovery retains the flush's observed safe block in a local variable. Post-flush Sync must succeed; its provider failure cannot use the initial-sync fallback. The guarded cascade transaction refuses divergence or a missing rollback-safe checkpoint, requires the persisted safe head to reach the flush observation, and then applies the post-flush policy. It runs even if the refreshed danger verdict is `Safe`: a young unresolved suffix is still doomed after flushing. A crash or retry loses the observation, so another invocation must flush again. - -Flush changes only the wallet watermark locally. New divergence can be discovered only by Sync, and the next dispatch or guarded cascade checks it before repair. There is no additional inspection between Flush and Sync. The process lock and task-free startup exclude a competing local writer; revisit this sequencing if startup gains concurrent writers. - -Every repair is followed by a current inspection. A surviving view/clock refusal retries the boot; successful mutation alone does not authorize serving. The guarded Tip operations also recheck their policy at mutation time, because wall-clock aging can change a verdict without a database writer. A repair must commit an open Tip, and startup never starts a second repair in the same invocation. - -After a clean result, runtime preparation launches zero tasks. `admit_runtime` then applies the same dispatch to current facts. Only `Ready` mints `RuntimeAdmission`; any repair requirement or refusal drops the prepared resources and exits. Launch consumes the witness synchronously. This final check is necessary because preparation can outlive the freshness of the persisted L1 view. - -`preemptive.tla` covers slot/batch safety. `admission.tla` covers local terminal dominance, flush/sync prerequisites, loss of observations across retries/crashes, repair postconditions, and final admission soundness. The “everything past gold is doomed” argument remains external to both bounded models. - -### Startup observability - -Startup logs its selected action with `danger_status`, `danger_batch_index`, and `recovery_decision`, and records the invalidated batch indexes after repair. Errors retain the classified retry/refuse verdict and their diagnostic cause. The orchestrator owns restart policy and alert routing. - -### L1 view freshness - -The safety policy does not branch directly on a provider-reachability boolean. Reachability is an execution concern: the initial Sync may fail while a warm persisted view remains usable, whereas a post-flush Sync failure must retry because Cascade requires a newly caught-up view. The decision primitive is the freshness of the L1 view recorded in SQLite plus the post-flush witness floor when one exists. - -The most common real-world trigger for `L1ViewStale` is a stalled RPC gateway: the provider answers, but its safe-head response stops advancing (a degraded upstream node, a load-balancer routing to a lagging replica, or a temporary indexing pause). The sequencer can't distinguish "fresh answer from a stalled view" from "L1 itself is unhealthy" without a second source of truth, so it treats both the same way: refuse to commit to soft confirmations until the recorded safe block is fresh again. - -**At startup**: the sequencer first inspects local terminal facts, then attempts the initial safe-head Sync, then inspects the persisted safe-block and progress timestamps. If the L1 timestamp is missing or older than `l1_read_stale_after_blocks * seconds_per_block`, `check_danger` returns `L1ViewStale` and startup retries. A baseline a full block-time or more ahead of `now` also yields `L1ViewStale`, but only after observed-safe checks have run. If those checks selected a repair, the repair completes and its mandatory next inspection applies the clock refusal. If the view is usable and fresh, observed-safe checks can route to recovery, and the batch-relative wall-clock estimate remains the final retry guard. - -**At runtime**: the `DangerDetector` polls `Storage::check_danger` on its cadence. The input reader records both the observed safe block timestamp and the local time at which the safe head last advanced. If safe-head observations stop advancing, either the global safe block timestamp crosses the read-staleness threshold (`L1ViewStale`) or a specific unresolved batch crosses the batch-relative adjusted threshold (`EstimatedBatchInDanger`). A backward clock step of a full block-time or more against either persisted baseline also produces `L1ViewStale` — evaluated after the observed arms — and saturation must never reinterpret such a regression as zero elapsed time; sub-block steps are quantization noise for the block-granular estimate and are tolerated. The detector then exits with `RecoveryRequired`, the orchestrator respawns, and startup re-runs the same check. The batch submitter never observes danger; this responsibility lives entirely with the detector. - -**Other workers during L1 outages**: the inclusion lane and API are purely local (SQLite) and continue operating. The input reader retries L1 polling with error logging. All L1-dependent workers log errors at the `error` level to alert operators. - -The `seconds_per_block` parameter (default: 12 for Ethereum) is configurable via `CARTESI_SEQUENCER_SECONDS_PER_BLOCK`. The L1 read-staleness threshold is configurable via `CARTESI_SEQUENCER_L1_READ_STALE_AFTER_BLOCKS`; its fixed default is independent of the margin and must remain strictly below the danger threshold (defaults and validation live in `sequencer/src/commands/config.rs`). These estimates are conservative — they may cause earlier detection if blocks are slower than assumed. This is correct: better to crash early than to issue doomed soft confirmations. - -## Dead Batches - -After cascade invalidation, submitted Pending batches (those with `w_nonce` assigned) are **dead batches**. They are still in the L1 mempool, competing with their flush no-op transactions. - -Two outcomes per dead batch, non-deterministic: - -- **Dead batch beats no-op**: lands on L1, scheduler sees it, rejects it (stale by inclusion, or nonce-poisoned by a preceding stale/missing batch) -- **No-op beats dead batch**: dead batch killed forever, scheduler never sees it (the scheduler skips the gap) - -A killed batch acts as **silent nonce poison**: the scheduler never sees it, so `schedulerExpected` stays stuck at its `batch_nonce`. All subsequent batches have wrong nonces. - -Dead batches occupy `w_nonce` slots strictly below `walletNonce`. Recovery batches occupy `w_nonce` slots at or above `walletNonce`. **No overlap.** This is why no mutual exclusion is needed between dead batches and recovery batches -- they live in non-overlapping `w_nonce` ranges. - -## Cockroach recovery (`setup --recovery`) - -Use [cockroach recovery](cockroach.md) when the database is lost or local state -is unusable, including after a sequencer bug. Fix the bug first, then rebuild in -a fresh data directory from a trusted, sufficiently advanced canonical -application checkpoint and L1. The recovery guide owns checkpoint requirements, -the replay procedure, and the resulting resume baseline. - -Plain `setup` also directs the operator to this path when it detects a previous -instance's batches past the checkpoint: it refuses with exit `40` -(`EXIT_SETUP_NEEDS_RECOVERY`). - -## Canonical divergence (terminal, outranks every arm) - -Independent of the staleness machinery, the input reader's acceptance -simulation cross-checks every at/above-anchor **accepted** landing against the -local valid closed batch at that nonce (the content-identity check: -`keccak256` of the landed wire bytes vs the hash stamped at seal). A `Foreign` -(no local batch) or `Mismatch` (different bytes) outcome persists the -`canonical_divergence` marker in the same transaction as the sync that found -it. The freeze, its runtime reaction, the race bound, and the watchdog -boundary are owned by -[I15](../invariants.md#i15-divergence-marker-present--acceptance-frontier-frozen); -the check's completeness scope by [I9](../invariants.md). - -This page owns the recovery side. `check_danger` reports -`CanonicalDivergence` **ahead of every other arm**, so a respawn loop can never -route a known-diverged node into a provider call, batch-tree mutation, or admission. -Startup checks before its initial Sync, after that Sync, inside the post-flush -cascade transaction, and before admission. The guarded repair transactions -reassert the marker's absence and map it to terminal `Refuse`. - -The remedy is **cockroach recovery (wipe + rebuild from L1), never the -standard recovery on this page**: the cascade reconciles the batch tree's -*shape* under the assumption that accepted nonce N is our batch N — a content -mismatch means canonical state contains executed effects with no reliable -local source, so rebuild-from-L1 is the only honest repair. - -### Restore points and admission - -Every admitted database retains a rollback-safe application artifact: the -baseline before any local batch is accepted, or an accepted batch snapshot. -Startup may load a newer surviving optimistic snapshot to reduce replay, but -that snapshot alone cannot justify admission because recovery may remove it. -Once an accepted artifact exists, GC may retire baseline bytes while preserving -immutable baseline metadata and active leases. `admission.tla` calls this -`hasRecoveryCheckpoint`; artifact creation and GC remain outside that model. - -## Implementation Constraints - -These constraints were discovered during TLA+ model checking and are required for correctness: - -1. **`walletNonce` must NOT be reset during recovery.** Recovery batches must use `w_nonces` strictly past all dead batch slots. The flush consumes dead batch slots by advancing `nextL1Slot` up to `walletNonce`. Recovery starts fresh from there. - **Mechanism:** `walletNonce` is realized durably as the `wallet_nonce_watermark` singleton — the highest wallet nonce ever broadcast. Every broadcaster (the batch poster and the flusher's no-ops alike) commits `watermark = max(watermark, n)` power-loss-durably (`synchronous=FULL`) **before** sending at nonce `n` (write-before-broadcast; a crash between commit and send only over-covers — one wasted no-op). The flush's completion condition is `pending <= safe && safe >= watermark + 1`, so it cannot declare victory while any slot we ever used is unresolved — restoring this constraint against the local pool's volatile memory. The watermark is never reset and never lowered. - -2. **`SubmitBatch` must use `max(walletNonce, nextL1Slot)`.** Prevents assigning `w_nonce` values for slots L1 has already consumed. - -3. **`SubmitBatch` must assign ALL pending batches at once, in spine-position order.** If batches are submitted individually, a flush-win can bump one batch's `w_nonce` past a later batch's, violating the spine ordering invariant. - -4. **Wall-clock freshness when the L1 view stops advancing.** The input reader records the L1 safe block timestamp and the local last-safe-head-progress time. `Storage::check_danger` first refuses on an old or missing safe block timestamp; a clock a full block-time or more out of step with either persisted baseline also refuses, but only after the observed-safe checks (sub-block skew is tolerated as quantization noise). Only a usable clock reaches the unresolved-batch estimate (`elapsed / seconds_per_block`). Without these checks, an L1 outage or a large backward clock step can silently push batches past the danger zone while the DB-based safe-block number remains frozen. - -5. **The accepted-frontier cache persists acceptances, not scan progress.** `safe_accepted_batches` stores the scheduler-accepted prefix and resumes from the latest accepted safe input. Rejected batch-submitter inputs after that frontier can be rescanned on later safe-head syncs until a later batch is accepted. This is a performance tradeoff, not a correctness bug: recovery batches can reuse a scheduler nonce after earlier rejected rows, so a separate persistent scan cursor would need careful nonce-reuse tests before being introduced. +Here Latest, Pending, and Safe are account transaction counts, not block +numbers; absent `W` contributes a lower bound of zero. An original batch or a +no-op may win each slot. Completion means every covered slot is consumed at +safe depth, even if the local node forgot an original transaction. It does not +require erasing that transaction's bytes from every mempool. + +The flusher returns the **safe block number** at which it observed completion. +Startup keeps it only in the current call; a crash or retry loses that +observation and the next attempt flushes again. Flush changes no local recovery +facts except the wallet watermark. + +**Sync through that observation.** Sync ingests safe InputBox events and updates +the local scheduler-acceptance projection. It does not query a canonical +application machine. A provider failure here retries the boot; there is no +fallback to the pre-flush view. + +**Cascade under an immediate SQLite transaction.** Its guard requires no +canonical divergence, a rollback-safe checkpoint, and a persisted safe head at +least as high as the flush observation. Then choose the pivot: + +- First valid closed batch beyond the accepted frontier, regardless of age. +- If none remains, the Tip only if it has reached `danger_threshold`. +- Otherwise invalidate nothing, retaining a fresh Tip or opening a missing one. + +The cascade deliberately runs even if refreshed danger is now `Safe`. The +closed-suffix policy follows from having completed flush and sync; it does not +repeat the trigger test. There is no extra inspection between flush and sync: +the process lock and absence of workers exclude competing local writers, and +sync is the step that can discover new divergence. Revisit this sequence if +startup gains concurrent writers. + +Flush completion depends on L1 progress. Replacement attempts can be rejected +or remain uncompetitive, and provider failures can interrupt the attempt. +Retries preserve safety but establish no recovery deadline. Pricing and its +accepted liveness limits belong to the [L1 fee policy](../l1-fee-policy.md). + +### Open Tip: repair without flushing + +An open Tip has never been submitted, so invalidating it creates no L1-slot +race. `RecoverTip { N }` rechecks divergence, checkpoint availability, and +**exactly** `TipInDanger(N)` inside its transaction, then invalidates that Tip +and opens a fresh one. It does not invalidate closed batches or fall back to +repairing a changed decision. + +The threshold is a policy choice: retaining an aging Tip would make the +runtime detector stop service again. Waiting for `MAX_WAIT_BLOCKS` would keep +the same suspected prediction alive without solving that cycle. + +`EnsureOpenTip` is a separate action. Its transaction requires `Safe`, no Tip, +no divergence, and a rollback-safe checkpoint. It opens the Tip without +invalidating history. Guarded writes matter even without concurrent writers: +wall-clock aging alone can change the decision after inspection. + +### Atomic history change and replay + +Invalidation, history rewind, generation change, and Tip creation share one +transaction. Invalidation removes the suffix's `application_inputs` projection; +raw source facts remain. `RecoveryGeneration` increments once iff at least one +valid batch was invalidated. Failed reopening rolls all of this back. + +The new Tip follows the latest surviving batch, or uses the immutable root +anchor if none survives. It attributes direct inputs after the surviving +frame's drain boundary, with the era baseline block as a floor. Storage records +those application entries; the launched lane executes them during catch-up. +This prevents a restored prefix's directs from being executed twice. + +Recovery requires the latest accepted batch snapshot, or the era baseline +before any local batch is accepted. A newer surviving optimistic snapshot can +reduce replay, but cannot replace that rollback guarantee. Snapshot publication, +artifact validation, leases, and GC are owned by the +[snapshot lifecycle](../snapshots/lifecycle.md); history coordinates and +subscription behavior by the [API contract](../../README.md). + +## Why the closed-suffix policy is safe + +**Settling slots removes the zombie race.** Before the flush, an old batch may +still win an L1 slot after local invalidation. Reusing its scheduler nonce too +early can let later old batches execute against the replacement branch. The +[historical counterexample](history/README.md) demonstrates this failure. +Detecting danger before settlement is necessary; mutating the closed suffix +before settlement is the unsafe step. + +After completion, the original transactions cannot newly win those consumed +slots on descendants of the observed safe chain. Sync through the observation +accounts for the originals that did win. Replacements use later wallet slots +and reuse only the scheduler nonces beyond the accepted prefix. This relies on +the trusted, consistent L1 view and dedicated submitter key in the +[threat model](../threat-model/README.md), and on every broadcaster preserving +the watermark. + +**A skipped batch does not advance the scheduler nonce.** When nonce `N` arrives +stale, its frames are skipped and later `N+1`, `N+2`, … envelopes encounter a +nonce mismatch. The overdue-direct backstop still runs before envelope +classification, so “skipped batch” does not mean the whole input has no state +effect. A missing batch whose slot was consumed by a no-op also leaves the +expected nonce unchanged. Later input cannot retroactively make already +rejected envelopes execute. + +**Discarding the entire remaining closed suffix is a convergence policy.** It +can include rejected landings, no-op-replaced transactions, and batches never +submitted at all. Some of that work could theoretically be resubmitted fresh; +“everything past Gold is doomed” is not a general impossibility proof. Recovery +sacrifices it to avoid preserving a partly submitted suffix and restarting into +the same danger/flush cycle. The cost is invalidated soft confirmations. + +If every closed batch became accepted, an aging Tip can still need repair: +its first frame may share the preceding batch's safe block, while its age is +measured at the later post-flush head. A fresh Tip survives without a generation +change. Thus flush alone does not imply invalidation. + +**Content identity is a prerequisite.** The input reader checks at/above-anchor +accepted wire bytes against the local valid closed batch. A foreign or different +payload persists divergence and freezes the acceptance frontier. Startup checks +that marker before L1 access, after sync, inside repair transactions, and before +admission. Repairing the tree's shape cannot recover missing canonical effects; +investigate the fault and use [cockroach recovery](cockroach.md). Check scope +and enforcement are owned by [I9 and I15](../invariants.md). ## Formal Verification -The recovery design is verified with two complementary bounded TLA+ models. [`preemptive.tla`](preemptive.tla) owns slot/batch safety; [`admission.tla`](admission.tla) owns startup reduction and runtime admission. An alternative optimistic batch design is preserved in [`history/optimistic.tla`](history/optimistic.tla). - -**Scope and limitations**: these are bounded safety models. They exhaustively check all reachable states within the configured bounds but do not prove liveness or model concrete timing margins. The admission model includes abstract owner loss/crash with fresh-attempt restart (there is no admission state machine to model); the slot model does not model crash/restart and relies on SQLite atomicity for its implementation mapping. +Two bounded TLA+ models check complementary safety obligations. They are not +a refinement proof of the Rust implementation, a proof of their composition, +or a liveness guarantee. Read both before changing recovery code. -### `preemptive.tla` -- Slot-level safety under adversarial flush +### `preemptive.tla`: batches and wallet slots -Models the core slot-level mechanics of preemptive recovery. At every `w_nonce` slot, L1 non-deterministically includes the spine batch OR a flush no-op (killing the batch). This covers the case where the frontier batch itself is killed during flush. The model also treats the open Tip's `safe_block` as meaningful, so it can explicitly recover an aging Tip that has no L1 footprint yet. +[`preemptive.tla`](preemptive.tla) models safe-block advancement, wallet-slot +competition between batches and no-ops, scheduler acceptance, and branch +invalidation. Its `Inv` checks `ZombieSafety` at every reachable state: +`schedulerExpected = CountGold(spine)`. It also checks batch-nonce contiguity, +invalid-branch ancestry, wallet-slot uniqueness, and L1/scheduler cursor bounds. -The model is a **safety over-approximation for the actions it shares with the implementation**: it allows `AdvanceTip` and `SubmitBatch` to interleave freely with recovery, which the real protocol prevents (the sequencer goes offline). This makes the proof stronger -- if `ZombieSafety` holds under more interleavings, it holds under fewer. However, the over-approximation claim does **not** hold action-for-action — two implementation actions sit *outside* the model's transition set: (1) the model discards an aging Tip only at `MAX_WAIT_BLOCKS`, while the implementation invalidates at `danger_threshold` (= `MAX_WAIT − MARGIN`); (2) the model's `Resolve` has no case for a killed-Pending frontier (it relies on resubmission until the frontier is Silver), while guarded post-flush Cascade invalidates killed Pendings unconditionally. Their safety rests on the external arguments above. Sequential startup ordering is intentionally delegated to `admission.tla` rather than cross-producting this already-large slot model. +Several details must not be read as literal production behavior: -**Verified**: 157M states, 0 violations. +| Model | Production mapping or limit | +|---|---| +| A Gold genesis sentinel at nonce zero | Production opens a parentless root at its stored anchor, without a submitted sentinel. Root/anchor cases are tested in Rust. | +| `SubmitBatch` assigns the pending suffix with `max(walletNonce, nextL1Slot)` | The poster derives the suffix and Latest account nonce, and raises the durable watermark before sending. The model expression is not a Rust nonce-allocation recipe. | +| Tip advancement and submission can interleave with recovery; dead batches can race after model invalidation | Production stops workers and settles covered slots before the closed cascade. These additional modeled interleavings do not establish coverage of different production actions. | +| `Resolve` handles a stale Silver frontier or a Tip at `MAX_WAIT_BLOCKS` | Production also invalidates a killed/unsubmitted closed suffix after flush and repairs a Tip at the earlier danger threshold. Those actions need the arguments above and Rust tests. | -| Invariant | Meaning | -|-----------|---------| -| ZombieSafety | `schedulerExpected = CountGold(spine)` -- scheduler accepts exactly the Gold prefix | -| BatchNoncesContiguous | Batch nonces are 0..N-1 for non-Tip spine | -| InvalidOnlyOnGold | Dead branches only hang off Gold nodes | -| L1WNonceUnique | No two L1 entries share a `w_nonce` | -| L1BeforeCursor | All L1 entries have `w_nonce < nextL1Slot` | -| SchedulerBehindL1 | Scheduler cursor doesn't pass L1 cursor | -| DeadNotYetIncluded | Dead batches have `w_nonce >= nextL1Slot` | +The model's `Gold`, `Silver`, `Bronze`, `Pending`, and `Tip` colors describe +stages of inclusion and acceptance. `Gold* Silver* Bronze* Pending* Tip` is +**not** an invariant: flushing can leave a killed Pending before a surviving +Silver. Do not build implementation assumptions on that ordering. -### `admission.tla` -- Sequential startup and admission +The configured finite bounds are in [`preemptive.cfg`](preemptive.cfg). +The model has neither crash/restart nor the wall-clock freshness policy. -Models the local terminal gate, initial Sync with warm-provider fallback, repair selection, guarded Tip repair, Flush → Sync → Cascade with an ephemeral observation, Sync-discovered divergence, post-repair checking, task-free preparation, and final current admission. Retry, refusal, and crash return to a fresh attempt over surviving durable facts; terminal-fault telemetry does not gate the next boot. +### `admission.tla`: startup and permission to launch -**Verified**: 554 generated states, 155 distinct states, depth 11, 0 violations. +[`admission.tla`](admission.tla) models the local terminal gate, initial-sync +fallback, at most one repair, flush observation and mandatory sync, guarded +cascade, post-repair inspection, task-free preparation, and final admission. +Retry, refusal, or owner loss starts a fresh attempt over surviving durable +facts; the flush observation and admission witness do not survive. -The invariants cover runtime admission soundness, terminal dominance, repair preconditions, the caught-up post-flush view, and observation scope across attempts. They express these safety obligations independently of the number of inspections. Concrete SQLite transaction atomicity and guards are tested in Rust; the batch spine remains in `preemptive.tla`. +Its invariants cover terminal dominance, repair preconditions, a caught-up +post-flush view, and admission soundness. It abstracts successful repair as +producing a Tip; concrete transactions and rollback are checked by Rust tests. +Neither model covers external era/generation metadata, the application-input +projection, or snapshot artifact/lease/GC durability. Those obligations remain +in storage constraints, tests, and the [snapshot lifecycle](../snapshots/lifecycle.md). -### Running the spec +Run the configured checks with: ```bash tlc -workers auto -deadlock docs/recovery/admission.tla -tlc -workers auto -deadlock docs/recovery/preemptive.tla # ~90s +tlc -workers auto -deadlock docs/recovery/preemptive.tla just -f docs/recovery/justfile check-all ``` -Bounds are in `admission.cfg` and `preemptive.cfg`. The `MaxWalletNonce` bound keeps the slot model finite (kill/resubmit cycles generate new `w_nonce` values). Increase bounds for higher confidence at the cost of longer runtime. +## Implementation and test map + +| Concern | Owner and useful tests | +|---|---| +| Startup dispatch, error classification, final admission | [`recovery/mod.rs`](../../sequencer/src/recovery/mod.rs); procedure tests substitute only L1 sync/flush, keeping real SQLite inspections and repairs. | +| Detection, mutation guards, pivot and atomic cascade | [`storage/recovery.rs`](../../sequencer/src/storage/recovery.rs), [`recovery_tests.rs`](../../sequencer/src/storage/recovery_tests.rs); exact Tip guard, safe-view floor, unconditional post-flush policy, generation rollback, root nonce, and direct replay. | +| Observed danger versus estimates, accepted frontier and content identity | [`storage/l1_submission.rs` tests](../../sequencer/src/storage/l1_submission.rs), [`safe_accepted_batches.rs`](../../sequencer/src/storage/safe_accepted_batches.rs); stale-view precedence, clock faults, reused nonces, divergence freeze. | +| Slot settlement and broadcast coverage | [`recovery/flusher.rs`](../../sequencer/src/recovery/flusher.rs), [`l1/watermark.rs`](../../sequencer/src/l1/watermark.rs), [`submitter/poster.rs`](../../sequencer/src/l1/submitter/poster.rs). | +| Task-free preparation, launch, restore and catch-up | [`commands/run/`](../../sequencer/src/commands/run/), [`inclusion_lane/mod.rs`](../../sequencer/src/ingress/inclusion_lane/mod.rs). | + +The accepted-frontier cache stores acceptances, not scan progress. Rejected +inputs after the frontier may be rescanned on later syncs; a separate persistent +cursor would need nonce-reuse reasoning and tests. This is a storage performance +tradeoff, not an extra recovery phase or a modeled TLA+ invariant. diff --git a/docs/recovery/history/README.md b/docs/recovery/history/README.md index 74e173e1..c4ea14ed 100644 --- a/docs/recovery/history/README.md +++ b/docs/recovery/history/README.md @@ -1,56 +1,77 @@ # Recovery Design History -This directory preserves the optimistic recovery design -- an alternative to the preemptive approach documented in the parent [`README.md`](../README.md). Both designs are sound. We preferred preemptive for its operational properties. +This directory preserves the **historical optimistic recovery design** and its +counterexample. It is not the production recovery procedure. The +[current recovery guide](../README.md) owns automatic recovery; the +[cockroach guide](../cockroach.md) owns manual rebuilding. -## The Optimistic Design +## The optimistic alternative -In the optimistic design, the sequencer keeps accepting user operations and building batches while recovery plays out in the background. If a batch goes stale, the system detects it when the batch becomes Silver (safe on L1), cascade-invalidates, and submits recovery batches -- all while the sequencer continues serving soft confirmations. +The sequencer would keep accepting user operations and building batches while +recovery ran concurrently. The retained [model](optimistic.tla) permits a +cascade only when the first unresolved batch is **Silver** (included in a safe +L1 block) and stale by its **inclusion block**. Recovery replaces the suffix and +resets the next wallet nonce to the next unconsumed L1 slot. Submitted batches +from the invalidated suffix may remain in the network as zombies competing with +new recovery batches. -The TLA+ spec [`optimistic.tla`](optimistic.tla) models this design with a scheduler, wallet nonces, zombie batches (invalidated batches still in the L1 mempool), and adversarial L1 inclusion. At each `w_nonce` slot where a zombie and a recovery batch compete, L1 non-deterministically picks one (wallet-nonce mutual exclusion). +The recorded bounded check reported 194M states with no invariant violations +after the Silver-only fix. This is model evidence, not a proof of production +recovery or its completion time. Bounds are in [`optimistic.cfg`](optimistic.cfg). -**Verified**: 194M states, 0 violations (after the Silver-only fix below). +## The counterexample: invalidating before slot resolution -## The Silver-Only Constraint +The rejected variant allowed an unresolved frontier to be invalidated based on +its current age, before its L1 outcome was settled. The danger was **cascading +and reusing wallet-nonce slots**, not detecting danger early. -Both designs share a critical constraint: **recovery must wait for the frontier batch to be Silver before cascade-invalidating.** - -This constraint was discovered through the optimistic model. The original design allowed staleness detection on Pending or Bronze batches (a "short-circuit" for faster recovery). TLA+ found a counterexample: - -Three batches with `MAX_WAIT_BLOCKS = 2`: +Take `MAX_WAIT_BLOCKS = 2` and three original batches: +```text +batch nonce 0 1 2 +safe_block 0 0 1 +wallet nonce 0 1 2 ``` -batch bn=0 bn=1 bn=2 -sb 0 0 1 -wn 0 1 2 -``` - -With `currentSafeBlock = 2`, `bn=1` is stale by current block, `bn=2` is fresh. If we cascade from `bn=1`, both become zombies. Recovery creates a new `bn=1` at `wn=1`. -At L1 slot 1, zombie `bn=1` and recovery `bn=1` compete (same `w_nonce`): +Assume batch 0 is already accepted. At `currentSafeBlock = 2`, batch 1 is old +enough to be stale if included now, while batch 2 is still fresh. If recovery +invalidates batches 1 and 2 while they are pending, it can submit a fresh +replacement batch 1 at wallet nonce 1. -- **Zombie wins**: scheduler sees it, stale, skip. Nonce poisoned. Safe. -- **Recovery wins**: zombie `bn=1` dies (never reaches L1). Recovery accepted. `schedulerExpected` advances to 2. Zombie `bn=2(wn=2)` is fresh (`inclusion_block - safe_block = 1 < 2`), matches expected nonce -> **accepted**. The scheduler executes invalidated batch data. +At L1 slot 1, the original and replacement compete: -The two protection layers (wallet-nonce mutual exclusion and nonce poisoning) undercut each other: mutual exclusion kills the batch that nonce poisoning needs. +- **Original wins:** the scheduler sees the stale batch and leaves its expected + batch nonce at 1. The original batch 2 then fails the nonce check. +- **Replacement wins:** the original batch 1 cannot land. The fresh replacement + advances the scheduler's expected nonce to 2. If the original batch 2 lands + in block 2, its age is `2 - 1 < 2` and its nonce matches: the scheduler accepts + data the sequencer already invalidated. -The fix: only detect staleness when the frontier is Silver (safe on L1, immutable). The scheduler is guaranteed to see it before any recovery batch. +Wallet-nonce mutual exclusion removed the stale batch that the nonce-poisoning +argument depended on. The retained optimistic model's `Resolve` therefore +requires a Silver frontier that is stale by inclusion: that original batch is +already on safe L1 and cannot be displaced by a replacement. -## Why We Chose Preemptive +## Why production uses preemptive recovery -Both designs are sound once Silver-only detection is enforced. The difference is operational: +Production closes intake and performs recovery offline. For closed-batch +recovery, the flush consumes every covered wallet-nonce slot at safe depth, +**whether the original transaction or a no-op wins**, then re-syncs the accepted +prefix before cascading. It does not require the original frontier batch to +become Silver. An unsubmitted open Tip has no wallet slot to settle. -**Both designs wait.** Any recovery design must wait for the frontier to become Silver before cascading. In the optimistic design, the sequencer keeps issuing soft confirmations during this wait -- confirmations that will be invalidated when the cascade fires. In the preemptive design, the sequencer goes offline before the cascade, so no doomed soft confirmations are issued. +This gives recovery a sequential procedure and stops new soft confirmations +while submission uncertainty is being resolved. The optimistic alternative +keeps serving through that interval and may add confirmations that a later +cascade revokes. -**Preemptive is simpler to reason about.** The optimistic design has concurrent actors: the batch submitter, the inclusion lane, L1 mempool competition, and recovery all interleave. The preemptive design is sequential: stop, flush, recover, resume. Each step has clear preconditions and postconditions. +The tradeoff is downtime. Flush completion requires L1 progress; fee headroom +does not guarantee replacement or establish a deadline. The +[current recovery guide](../README.md) owns the safety conditions, and the +[L1 fee policy](../../l1-fee-policy.md) owns the accepted liveness limits. -**Preemptive eliminates mempool races.** The flush resolves all `w_nonce` slot uncertainty before recovery runs. Recovery operates on fully-finalized L1 state. No zombie mutual exclusion needed. - -**The cost is downtime.** Preemptive recovery takes the sequencer offline for the duration of the flush + safe finality wait (~15-20 minutes on Ethereum). For a rare event (a batch approaching the 4-hour staleness deadline), this is acceptable. - -## Running the Spec +## Running the historical model ```bash -tlc -workers auto -deadlock docs/recovery/history/optimistic.tla # ~3min +tlc -workers auto -deadlock docs/recovery/history/optimistic.tla ``` - -Bounds are in `optimistic.cfg`. diff --git a/docs/snapshots/lifecycle.md b/docs/snapshots/lifecycle.md index f99ad0d3..0e9cd3f7 100644 --- a/docs/snapshots/lifecycle.md +++ b/docs/snapshots/lifecycle.md @@ -127,6 +127,9 @@ the enclosing dump directories recursively. This filesystem operation disposes of all checkpoint resources. Filesystem deletion failure leaves a harmless orphan for the startup sweep. The reverse ordering would leave a durable row pointing at missing state and is forbidden. Startup clears leases left by the dead process, -validates a rollback checkpoint, collects obsolete rows, and sweeps orphan -directories before workers start. Missing or corrupt referenced artifacts fail -loud; operational filesystem errors retain their normal error classification. +checks the rollback checkpoint's `info.toml` and format version, collects obsolete +rows, and sweeps orphan directories before workers start. Application restoration +runs afterward in the launched inclusion lane, before processing new user ops; +the metadata check does not validate the application bytes. Missing or corrupt +referenced artifacts fail loud when read or restored; operational filesystem +errors retain their normal error classification. diff --git a/sequencer/src/commands/config.rs b/sequencer/src/commands/config.rs index 3702bdf7..33837c7b 100644 --- a/sequencer/src/commands/config.rs +++ b/sequencer/src/commands/config.rs @@ -43,9 +43,9 @@ pub struct TimingArgs { /// The danger threshold is MAX_WAIT_BLOCKS minus this margin. /// Must be less than MAX_WAIT_BLOCKS (validated at startup). /// - /// Default 300 (~1h at 12s/block) is sized to give operators meaningful - /// runway to investigate before the system gives up on the current - /// batches — see `docs/recovery/README.md` "Step 1: Danger threshold". + /// Default 300 gives ~1h of headroom before canonical expiry at 12s/block. + /// Detection starts recovery immediately; this is neither an operator + /// grace period nor a completion deadline. See `docs/recovery/README.md`. #[arg( long, env = "CARTESI_SEQUENCER_PREEMPTIVE_MARGIN_BLOCKS", diff --git a/sequencer/src/commands/run/startup_hygiene.rs b/sequencer/src/commands/run/startup_hygiene.rs index 409acaaf..e6005f4b 100644 --- a/sequencer/src/commands/run/startup_hygiene.rs +++ b/sequencer/src/commands/run/startup_hygiene.rs @@ -1,7 +1,7 @@ // (c) Cartesi and individual authors (see AUTHORS) // SPDX-License-Identifier: Apache-2.0 (see LICENSE) -//! Startup clears stale leases, validates the rollback checkpoint, then collects +//! Startup clears stale leases, checks rollback checkpoint metadata, then collects //! obsolete snapshots and orphan directories before workers are admitted. use crate::commands::error::CommandError; diff --git a/sequencer/src/commands/run/workers.rs b/sequencer/src/commands/run/workers.rs index 7ce22577..9fde8394 100644 --- a/sequencer/src/commands/run/workers.rs +++ b/sequencer/src/commands/run/workers.rs @@ -184,7 +184,7 @@ impl PreparedRuntime { let dumps_dir = std::path::Path::new(&run_config.data_dir).join("dumps"); std::fs::create_dir_all(&dumps_dir)?; - // Validate the rollback artifact and collect obsolete snapshots before admission. + // Validate rollback checkpoint metadata and collect obsolete snapshots before admission. super::startup_hygiene::run_snapshot_hygiene(&mut storage, &dumps_dir)?; // Prepare every remaining fallible or awaited dependency before the diff --git a/sequencer/src/ingress/inclusion_lane/catch_up.rs b/sequencer/src/ingress/inclusion_lane/catch_up.rs index cfd7ce8c..1571437d 100644 --- a/sequencer/src/ingress/inclusion_lane/catch_up.rs +++ b/sequencer/src/ingress/inclusion_lane/catch_up.rs @@ -1,7 +1,7 @@ // (c) Cartesi and individual authors (see AUTHORS) // SPDX-License-Identifier: Apache-2.0 (see LICENSE) -//! Restore the application's committed history suffix before runtime admission. +//! Replay committed application history after launch, before processing new user ops. use std::path::PathBuf; diff --git a/sequencer/src/recovery/flusher.rs b/sequencer/src/recovery/flusher.rs index bf9cc6a5..ee4242fb 100644 --- a/sequencer/src/recovery/flusher.rs +++ b/sequencer/src/recovery/flusher.rs @@ -157,10 +157,10 @@ impl MempoolFlusher { /// Flush the mempool by submitting no-op transactions for unresolved /// nonce slots, then waiting until every slot we ever used is safe. /// - /// `watermark` is the persisted wallet-nonce watermark — the highest - /// nonce this deployment ever broadcast, or `None` if nothing was ever - /// broadcast (or no DB survives, the cockroach-recovery best-effort - /// case). The loop runs until + /// `watermark` durably covers every wallet nonce this deployment may + /// have broadcast. Write-before-send can cover an unused slot. `None` + /// means no covered broadcasts, or a lost DB in cockroach recovery's + /// best-effort flush. The loop runs until /// /// ```text /// pending <= safe && safe >= watermark + 1 diff --git a/sequencer/src/storage/ingress.rs b/sequencer/src/storage/ingress.rs index 095f1fa1..2d7c0909 100644 --- a/sequencer/src/storage/ingress.rs +++ b/sequencer/src/storage/ingress.rs @@ -97,7 +97,7 @@ impl Storage { /// warm resume and recovery batches use), so there is no cold-start drain. /// /// This unguarded form exists for test harnesses only. Production uses - /// `ensure_open_tip_for_recovery`, which reasserts the reducer facts in its + /// `ensure_open_tip_for_recovery`, which reasserts the startup facts in its /// write transaction. The lane only ever *loads* a Tip (fail-loud if /// absent); Cascade owns its own atomic reopen using the same mechanism. /// @@ -471,7 +471,8 @@ pub(super) fn open_fresh_tip_in_tx(tx: &Transaction<'_>) -> Result<()> { } /// Capture the unaccounted range before creating its new frame, then attribute -/// its external directs. Catch-up executes these rows before runtime admission. +/// its external directs. After launch, lane catch-up executes these rows before +/// processing queued user operations. fn insert_draining_tip_with_executions( tx: &Transaction<'_>, batch_index: Option, diff --git a/sequencer/src/storage/mutations.rs b/sequencer/src/storage/mutations.rs index e62cf6f4..8495b2ad 100644 --- a/sequencer/src/storage/mutations.rs +++ b/sequencer/src/storage/mutations.rs @@ -16,8 +16,8 @@ use super::l1_inputs::query_deployment_identity; use super::{DirectInputExecution, SafeInputRange}; /// Insert a new batch. Nonce is derived from `parent_batch_index`: -/// `parent.nonce + 1`, or 0 if `parent_batch_index` is None (genesis or -/// post-cascade torn-state new Tip). +/// `parent.nonce + 1`, or the deployment's anchor for a parentless root +/// (genesis, cockroach recovery, or a fully invalidated branch). /// /// If `batch_index_opt` is None, SQLite auto-assigns (highest existing +1). /// The explicit form is used only by `initialize_open_state` to pin the @@ -64,9 +64,7 @@ pub(super) fn insert_new_batch( fn compute_next_nonce(tx: &Transaction<'_>, parent_batch_index: Option) -> Result { match parent_batch_index { // A parentless root carries the deployment's batch-tree anchor nonce: - // 0 for a genesis deployment, N' for a cockroach-recovered one. This - // generalizes the old hard-coded 0; the `batch_tree_anchor` row defaults - // to 0, so genesis and post-cascade re-roots are unchanged. Mirrored by + // 0 for a genesis deployment, N' for a cockroach-recovered one. Mirrored by // `trg_enforce_nonce_contiguity`'s parentless arm. None => batch_tree_anchor_in(tx), Some(parent_bi) => { diff --git a/sequencer/src/storage/recovery.rs b/sequencer/src/storage/recovery.rs index 6b7dbd29..f09e5ebc 100644 --- a/sequencer/src/storage/recovery.rs +++ b/sequencer/src/storage/recovery.rs @@ -1,11 +1,10 @@ // (c) Cartesi and individual authors (see AUTHORS) // SPDX-License-Identifier: Apache-2.0 (see LICENSE) -//! Recovery writer: cascade-invalidates stale batches, opens recovery batches, -//! and composes the startup-recovery transaction. +//! Recovery storage: danger inspection, guarded suffix invalidation, and Tip creation. //! -//! See `docs/recovery/README.md` for the full design (batch tree, coloring, -//! nonce poisoning, TLA+ proof). This file's job is to enforce that design +//! See `docs/recovery/README.md` for the procedure, safety arguments, and +//! bounded model coverage. This file's job is to enforce that design //! locally — read the design first if you're touching this code. //! //! Free functions here are shared with the batch submitter @@ -40,8 +39,7 @@ use super::snapshot_dumps::has_rollback_safe_snapshot_in; /// Each variant maps to a distinct response in the startup recovery procedure: /// /// - `L1ViewStale` → retry boot. The L1 safe block is too old or unknown. -/// - `ClosedBatchInDanger(closed_idx)` → enter the phase-granular -/// Flush/Sync/Cascade sequence. +/// - `ClosedBatchInDanger(closed_idx)` → Flush → Sync → Cascade. /// - `TipInDanger(tip_idx)` → direct Tip recovery, no flush. The Tip has no L1 /// footprint, so we can invalidate it and open a fresh one without /// any L1 round-trip. @@ -88,7 +86,7 @@ pub enum DangerStatus { /// One transactionally consistent local view consumed by the startup /// recovery procedure. /// -/// Keeping these facts together is load-bearing: admission and recovery-phase +/// Keeping these facts together is load-bearing: admission and repair /// selection must not combine a danger verdict from one SQLite snapshot with /// Tip/snapshot/head facts from another. #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -100,7 +98,7 @@ pub(crate) struct RecoveryInspection { } /// A recovery mutation was refused because the transaction no longer -/// satisfies the phase selected by the reducer. +/// satisfies the selected startup action's preconditions. #[derive(Debug, thiserror::Error)] pub(crate) enum RecoveryMutationError { #[error(transparent)] @@ -114,15 +112,13 @@ pub(crate) enum RecoveryMutationError { }, #[error("cannot open the Tip without a recovery checkpoint")] MissingRecoveryCheckpoint, - /// The `EnsureOpenTip` phase found a valid open Tip already present. A + /// `EnsureOpenTip` found a valid open Tip already present. A /// stale no-Tip decision, not a danger change; unreachable under the /// process lock, and retryable if it ever fires. #[error("the Tip was already open when the EnsureOpenTip phase ran")] TipAlreadyOpen, - /// The `EnsureOpenTip` phase's own transaction left no valid open Tip - /// after opening one. Impossible by construction; refused rather than - /// committed, so the reducer's one cycle (Repaired → EnsureOpenTip → - /// Repaired) cannot spin on it. + /// `EnsureOpenTip` left no valid open Tip after opening one. This broken + /// postcondition must roll back and refuse startup. #[error("the EnsureOpenTip phase left no valid open Tip in its own transaction")] TipMissingAfterOpen, #[error( @@ -247,7 +243,7 @@ impl Storage { self.read(|tx| inspect_recovery_in(tx, protocol, now_ms)) } - /// Execute the reducer's `EnsureOpenTip` phase only if its local decision + /// Execute `EnsureOpenTip` only if its local decision /// still holds in the write transaction. pub(crate) fn ensure_open_tip_for_recovery( &mut self, @@ -272,11 +268,8 @@ impl Storage { return Err(RecoveryMutationError::TipAlreadyOpen); } open_fresh_tip_in_tx(&tx)?; - // Postcondition, enforced where it can be violated: this phase is the - // only edge back into `Repaired` without a Tip, so it must never - // commit without one. A violation is a typed refuse (exit 30), never - // a retry that would spin the reducer, and never a `debug_assert` - // that compiles out. + // A broken postcondition must roll back this transaction and refuse + // startup, including in release builds. if !has_valid_open_batch(&tx)? { return Err(RecoveryMutationError::TipMissingAfterOpen); } @@ -284,7 +277,7 @@ impl Storage { Ok(()) } - /// Execute the reducer's `RecoverTip` phase only while the same Tip is + /// Execute `RecoverTip` only while the same Tip is /// still the observed-danger arm in the write transaction. pub(crate) fn recover_aging_tip_for_recovery( &mut self, @@ -312,7 +305,7 @@ impl Storage { Ok(invalidated) } - /// Execute the reducer's `Cascade` phase. The ephemeral flush witness is + /// Execute post-flush Cascade. The boot-local flush observation is /// represented by its observed safe-block floor; this transaction /// reasserts both I15 and the post-flush resync coherence check /// immediately before changing the batch tree. @@ -345,7 +338,8 @@ impl Storage { } /// Mark a single batch as invalid. Test-only seeder — production code goes - /// through [`Storage::recover_post_flush`] or [`Storage::recover_aging_tip`]. + /// through [`Storage::recover_post_flush_for_recovery`] or + /// [`Storage::recover_aging_tip_for_recovery`]. /// Idempotent: leaves already-invalid rows alone. #[cfg(test)] pub(crate) fn insert_invalid_batch(&mut self, batch_index: u64) -> Result<()> { @@ -368,7 +362,7 @@ impl Storage { /// Test-only unguarded primitive; production calls /// [`Storage::recover_aging_tip_for_recovery`], which transactionally - /// reasserts the exact reducer decision. Design rationale on + /// reasserts the exact startup decision. Design rationale on /// [`recover_aging_tip_inner`], the shared body. #[cfg(test)] pub fn recover_aging_tip(&mut self, danger_threshold: u64) -> Result> { @@ -441,126 +435,32 @@ fn refuse_divergence(danger: DangerStatus) -> std::result::Result<(), RecoveryMu // ── Free functions used by both recovery and the batch submitter ────────── -/// Cascade the non-gold suffix and open a fresh recovery batch (the shared -/// body behind [`Storage::recover_post_flush_for_recovery`], which the -/// reducer reaches after carrying a Flush witness through a caught-up -/// Sync). Homed here, not on the test wrapper, so rustdoc builds it and a -/// wrapper cleanup cannot delete the design record. +/// Discard the entire non-accepted closed suffix after flush and caught-up Sync. +/// This is a convergence policy: replaced or never-submitted work could be +/// submitted fresh, but preserving it can re-enter the same danger/recovery cycle. +/// The caller must settle wallet slots and refresh acceptance through the flush +/// observation; the production guard checks that the local view caught up. /// -/// # The "everything past gold is doomed" rule -/// -/// At this point the gold frontier is at its maximum extent: every -/// submitted batch has either been accepted (gold) or rejected by the -/// scheduler simulation (Silver-stale, since nonce-mismatch is impossible -/// at the frontier under self-trust), or its tx was killed by a flush -/// no-op (Pending, no `safe_input`). All three non-gold states are doomed: -/// -/// - **Silver-stale:** scheduler skipped it; downstream batches are -/// nonce-poisoned. -/// - **Pending:** the original L1 tx is dead. Re-submission could in -/// principle land fresh, but the *next* recovery cycle's flush would -/// compete with the resub at its new wallet-nonce slot and the bumped -/// no-op typically wins. The system would loop until current staleness -/// crossed `MAX_WAIT_BLOCKS`. Cascading now converges in one cycle. -/// -/// So once we've committed to recovery (the danger detector tripped, the -/// flush ran), the right move is to cascade the entire non-gold suffix -/// and open a fresh recovery batch. -/// -/// Three aftermath shapes: -/// -/// 1. **Everything worked:** all in-flight batches landed fresh and were -/// accepted. Gold extends to the last submitted batch; no first -/// non-gold closed. (See "Tip handling" below for the subtle subcase.) -/// 2. **Mixed:** some landed (stale or poisoned), some replaced. First -/// non-gold closed is either Silver-stale or Pending. Cascade from -/// there; the `batch_index >= N` rule catches the rest of the suffix -/// including the open Tip. -/// 3. **All replaced:** flush no-ops won every race. Gold doesn't -/// advance; first non-gold closed is the very first non-accepted batch. -/// -/// # Tip handling -/// -/// In cases (2)/(3) the cascade catches the Tip via `batch_index >= N`. -/// In case (1), there's no closed pivot — but the Tip can still be in -/// the danger zone: -/// -/// When the lane rotates a batch without a safe-block advance between -/// frames (e.g. immediately after init, when both share the bootstrap -/// `safe_block`), the Tip's `first_frame.safe_block` equals the closed -/// batch's. The closed batch can become gold by inclusion-staleness -/// (`inclusion_block - first_frame < MAX_WAIT`) while the Tip's age, -/// computed against `current_safe_block` after the flush wait, has -/// crossed `danger_threshold`. Pure monotonicity (`S_tip ≥ S_closed`) doesn't -/// rule this out — equality is allowed. -/// -/// So in the no-pivot branch we additionally check the Tip against -/// `danger_threshold` (the same threshold that would have triggered -/// recovery had the Tip been a closed batch). We're already committed -/// to recovery; the Tip is past gold; if it's also in the danger zone, -/// cascade it and open a fresh one. -/// -/// # Atomicity -/// -/// Runs as a single SQLite write transaction. On crash mid-way, the -/// txn rolls back; on commit, the cascade and the recovery batch open -/// land together. Idempotent on re-run because `valid_*` views filter -/// out already-invalidated rows. -/// -/// # Precondition -/// -/// The caller MUST have just synced L1 state via -/// [`Storage::append_safe_inputs`]; the gold frontier in -/// `safe_accepted_batches` must reflect the latest safe head. Otherwise -/// the cascade may invalidate batches that haven't yet had a chance to -/// be processed by the scheduler simulation. +/// If no closed pivot remains, only an aging Tip is invalidated. A closed batch +/// can have landed fresh while the Tip, even with the same first-frame clock, +/// has since crossed the danger threshold. Cascade and reopening share `tx`. /// /// Returns the newly-invalidated batch indices (empty if none). fn recover_post_flush_inner(tx: &Transaction<'_>, danger_threshold: u64) -> Result> { // Path 1: any closed batch past gold cascades unconditionally. let pivot = match first_non_gold_closed_batch(tx)? { Some(batch_index) => Some(batch_index), - // Path 2 (corner case): all closed are gold, but the Tip might be - // in the danger zone — see `recover_post_flush` doc on Tip handling. + // All closed batches are accepted; the Tip can still have aged. None => find_tip_batch_in_danger(tx, danger_threshold)?, }; cascade_and_reopen(tx, pivot) } -/// Cascade the open Tip if its first frame has aged past -/// `danger_threshold` (the shared body behind -/// [`Storage::recover_aging_tip_for_recovery`]). Homed here, not on the -/// test wrapper, so rustdoc builds it and a wrapper cleanup cannot delete -/// the design record. -/// -/// # Why a threshold here, but no closed-frontier check -/// -/// Outside a flush path, closed batches past the gold -/// frontier (if any) might still be in their natural lifecycle — -/// pending in the mempool, recently included, awaiting safe finality. -/// Cascading them would prematurely abort their progression. -/// -/// The Tip is different: it has no L1 footprint at all (no `w_nonce`, -/// no `safe_input`), so there's no L1 outcome to wait on. Once its -/// first frame has aged into the danger zone, the rule "everything -/// past gold is bad once we're committed to recovery" applies, and in -/// the `RecoverTip` path startup is already committed. -/// -/// # Threshold = danger_threshold, not MAX_WAIT -/// -/// We use `danger_threshold` (= `MAX_WAIT_BLOCKS - margin`) rather than -/// `MAX_WAIT_BLOCKS`. The Tip threshold is the same one that would -/// trigger the recovery cycle had the Tip been a closed batch. If the -/// Tip is past that threshold, the next danger detector tick after -/// resume would re-trip on the Tip's eventual first close + submission -/// anyway (the closed batch would inherit its first frame's safe_block). -/// Cascading now saves the cycle. -/// -/// # Precondition -/// -/// As with [`Storage::recover_post_flush`], the caller must have synced -/// L1 state. (Threshold comparison reads `current_safe_block` from -/// `l1_safe_head`.) +/// Discard only the aging Tip. It has no L1 footprint, so no flush is required. +/// Using `danger_threshold` avoids restarting with the same age that triggered +/// recovery; it is a policy threshold, not proof of canonical staleness. +/// The production caller rechecks the exact `TipInDanger` decision against the +/// current local inspection before entering this shared body. /// /// Returns the newly-invalidated batch indices (empty if Tip is fresh, /// `[tip_index]` when the Tip was cascaded). @@ -604,15 +504,12 @@ fn cascade_and_reopen(tx: &Transaction<'_>, pivot: Option) -> Result=`, not `>`: `frontier_nonce` is the *next-expected* nonce -/// (`latest_accepted.nonce + 1`), so the actual cascade-pivot batch carries -/// `nonce == frontier_nonce`. Using `>` would skip it. +/// (`latest_accepted.nonce + 1`, or the anchor before any acceptance), so the +/// actual cascade-pivot batch carries `nonce == frontier_nonce`. Using `>` +/// would skip it. /// -/// On the valid path, batch nonces are contiguous (enforced by the -/// `trg_enforce_nonce_contiguity` trigger), so the first match always has -/// `nonce == frontier_nonce`. We don't double-check that invariant here — -/// the trigger is the source of truth (see AGENTS.md "Self-trust": no -/// defense-in-depth checks against the sequencer's own bugs). Returns -/// `None` if all closed batches are gold. +/// Valid-path nonce contiguity (I16) makes the first match exactly +/// `frontier_nonce`. Returns `None` if all closed batches are accepted. fn first_non_gold_closed_batch(conn: &Connection) -> Result> { let frontier = frontier_nonce(conn)?; let batch_index: Option = conn @@ -629,13 +526,10 @@ fn first_non_gold_closed_batch(conn: &Connection) -> Result> { /// Either the closed-frontier batch or the Tip, whichever (if either) has /// aged past `threshold` against `current_safe_block`. Used by /// [`Storage::check_danger`]'s wall-clock-adjusted arm, where the dispatch -/// is the same (`Refuse`) regardless of which one fired. +/// is the same (`Retry`) regardless of which one fired. /// /// Closed-frontier wins: frame `safe_block`s are non-decreasing along the -/// spine, so the closed frontier is at least as *old* as the Tip — whenever -/// the Tip is in danger, the closed frontier is too, and cascading from the -/// closed batch covers the Tip via `batch_index >= N`. (This ordering is -/// also determines which batch snapshots remain valid.) +/// spine, so an existing closed frontier is at least as old as the Tip. /// /// Reads `safe_accepted_batches`, which is maintained atomically with each /// [`Storage::append_safe_inputs`] call. @@ -650,14 +544,9 @@ pub(super) fn find_first_batch_in_danger(conn: &Connection, threshold: u64) -> R /// than `current_safe_block - threshold`. Returns `None` if no such batch /// exists. /// -/// Why look only at the frontier batch, not "every batch past gold"? -/// `safe_accepted_batches` is updated atomically with each safe-head advance -/// (see [`super::safe_accepted_batches`]) and walks the spine until it hits -/// a barrier — a stale batch, or a missing slot the scheduler can't bridge. -/// So the first batch past the frontier IS the barrier; downstream batches -/// are nonce-poisoned by definition (a stale frontier ⇒ scheduler skips ⇒ -/// every later batch arrives at an unexpected nonce). Looking further is -/// redundant. +/// First-frame clocks are non-decreasing along the valid path (I3), so the +/// earliest non-accepted closed batch is at least as old as its successors. +/// Checking younger batches cannot reveal danger that this check missed. /// /// Does NOT consider the Tip — the Tip has no L1 transaction, so it's not /// part of the closed-frontier-staleness category. From 3ef56ec30e685792b5d869d25a20baa3d69b3548 Mon Sep 17 00:00:00 2001 From: gcdepaula Date: Thu, 17 Sep 2026 15:33:18 -0300 Subject: [PATCH 14/29] docs: align history snapshots and watchdog contracts --- AGENTS.md | 2 +- README.md | 38 ++- docs/plans/2026-07-coordination-tracks.md | 66 ++-- .../2026-07-track3-feed-replay-design.md | 150 ++-------- docs/plans/2026-07-track6-dump-api-design.md | 2 +- docs/plans/2026-08-authority-boundary-adr.md | 4 +- docs/plans/application-history.md | 92 ------ docs/protocol/application-contract.md | 45 ++- docs/protocol/application-history.md | 150 ++++++++++ docs/protocol/scheduler-semantics.md | 2 +- docs/recovery/cockroach.md | 2 +- .../2026-09-09-application-lane-dex-review.md | 2 +- docs/review/register.md | 6 +- docs/snapshots/README.md | 20 +- docs/snapshots/format.md | 135 ++------- docs/watchdog/README.md | 283 +++++++++--------- docs/watchdog/design-notes.md | 6 +- docs/watchdog/getting-started.md | 34 ++- docs/watchdog/operator-deployment.md | 106 ++++--- 19 files changed, 525 insertions(+), 620 deletions(-) delete mode 100644 docs/plans/application-history.md create mode 100644 docs/protocol/application-history.md diff --git a/AGENTS.md b/AGENTS.md index ba2494d5..a64aae4b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -445,7 +445,7 @@ work reaches another boundary. | Application implementation, execution, or native integration | [Application contract](docs/protocol/application-contract.md) — determinism, progress, failure, capacity, and checkpoints; [C binding](docs/protocol/c-application-binding.md) for native engines. | | Automatic recovery or danger detection | [Automatic recovery](docs/recovery/README.md), then [preemptive.tla](docs/recovery/preemptive.tla) and [admission.tla](docs/recovery/admission.tla) — repair ordering and the models' bounded guarantees. | | Manual rebuild after lost state or a sequencer bug | [Cockroach recovery](docs/recovery/cockroach.md) — trusted checkpoint, fixed input boundary, and fresh baseline. | -| API, subscriber replay, or application-history coordinates | [README API contract](README.md#api) — wire behavior; [application history](docs/plans/application-history.md) — era, generation, offsets, and recovery boundaries. | +| API, subscriber replay, or application-history coordinates | [README API contract](README.md#api) — wire behavior; [application history](docs/protocol/application-history.md) — era, generation, offsets, and recovery boundaries. | | Snapshots, restart, export, retention, or watchdog checkpoints | [Snapshot lifecycle](docs/snapshots/lifecycle.md) — durable publication, accepted comparison points, leases, and GC; [wallet format](docs/snapshots/format.md) when changing wallet bytes. | | Trust boundaries, provider behavior, or hostile L1 input | [Threat model](docs/threat-model/README.md) — actor assumptions, supported failures, and residual risks. | | Submission fees or oracle pricing | [L1 fee policy](docs/l1-fee-policy.md) — estimation and replacement limits; [threat-model actor table](docs/threat-model/README.md#actors-and-trust) — oracle source and outage assumptions. | diff --git a/README.md b/README.md index 68c1a0a9..8dfc7cfe 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Sequencer -A sequencer for Cartesi app-specific rollups. Provides low-latency soft confirmations for user operations, posts them to L1 in batches, and maintains a deterministic replay feed that matches the application's final execution order. +A sequencer for Cartesi app-specific rollups. Provides low-latency soft confirmations for user operations, posts them to L1 in batches, and exposes its current application execution order for replica replay. **Security-critical infrastructure.** Handle every change with the care financial systems demand. @@ -86,7 +86,7 @@ The sequencer is designed to handle: - **L1 provider outages** — workers retry with exponential backoff. The inclusion lane and API continue operating locally. A wall-clock fallback detects when an outage pushes batches into the danger zone. - **Undiagnosed interruptions (OOM, SIGKILL, reboot)** — restart can recover automatically: every boot derives any required recovery from SQLite and L1 safe state through startup recovery, never assuming the previous exit was clean. Terminal errors returned through a command bracket best-effort record their cause in `terminal_faults`; terminal runtime aborts leave only process diagnostics. - **Extended downtime** — startup syncs to the current L1 safe head, flushes if needed, and recovers before admission. A terminal exit requires operator investigation; rebuilding untrustworthy state follows the cockroach recovery procedure above. -- **Adversarial L1 mempool** — block builders and private mempools are treated as adversarial. The recovery flusher consumes every pending nonce slot with a no-op so delayed "zombie" submissions cannot land later. +- **Adversarial L1 mempool** — block builders and private mempools are treated as adversarial. Recovery waits until every covered wallet-nonce slot is consumed at safe depth, whether the original transaction or a flush no-op wins, so delayed "zombie" submissions cannot land later. ## Interfaces @@ -96,7 +96,13 @@ Users submit signed operations via `POST /tx` (JSON). Operations are signed with ### Sequenced Transaction Feed -Subscribers connect via `GET /ws/subscribe?era_id=&recovery_generation=&next_input=` (WebSocket). The feed delivers all sequenced transactions (user ops + direct inputs) in deterministic order, matching the on-chain execution order. This is the primary interface for downstream consumers (frontends, indexers). The endpoint is designed for a small number of indexer subscribers, which serve users directly. +Subscribers restore an HTTP snapshot, then use one WebSocket stream to replay +application inputs and follow the optimistic tip. Recovery can replace that +history; the snapshot's era, generation, and input count bind a resume request +to the state the consumer actually holds. The endpoint serves a small number of +infrastructure subscribers, which serve users directly. See the +[bootstrap workflow](docs/protocol/application-history.md#replica-bootstrap-and-resume) +and [wire contract](#api). ### Batch Submission @@ -187,6 +193,16 @@ Notes: - queue capacity is an internal runtime constant tuned alongside inclusion-lane chunking to absorb short bursts; if this starts triggering persistently, it is a signal to revisit runtime sizing or throughput rather than add another admission layer. - Browser wallets can call `POST /tx` and `GET /fee` from any origin with any request headers; preflight permits GET and POST and is cached for one hour. CORS is applied only to ingress. Egress routes remain operator-only and require network access controls. +Success response after inclusion: + +```json +{ + "ok": true, + "sender": "0x...", + "nonce": 0 +} +``` + ### `GET /fee` Fee quote for setting signed user-op `max_fee` before `POST /tx`. All three fields are log-space exponents (base 129/128), the same encoding as `max_fee`. Inclusion rejects any op with `max_fee` below the open-frame `fee`. @@ -205,7 +221,7 @@ Notes: ### `GET /ws/subscribe?era_id=&recovery_generation=&next_input=` -WebSocket stream of canonical application inputs, replaying from the inclusive +WebSocket stream of the current application history, replaying from the inclusive `next_input` offset and then following the optimistic tip. Fetch and restore `/latest_snapshot` first; its headers supply the complete subscription claim. After each successfully applied input at offset `X`, persist the claim with @@ -235,16 +251,6 @@ Message shapes: { "kind": "direct_input", "offset": 11, "sender": "0x...", "block_number": 123, "block_timestamp": 1700000000, "transaction_hash": "0x...", "payload": "0x...", "input_index": 42, "batch_nonce": 4 } ``` -Success response: - -```json -{ - "ok": true, - "sender": "0x...", - "nonce": 0 -} -``` - ### Operator snapshot endpoints (internal only) These serve application state to the operator's watchdog and indexers. @@ -265,7 +271,7 @@ api split lands). adding a coherent `checkpoint.toml` receipt with its L1 inclusion block and next batch nonce for trusted recovery. -All state/archive responses include `X-History-Era`, `X-Recovery-Generation`, +Successful state/archive downloads include `X-History-Era`, `X-Recovery-Generation`, and `X-Executed-Input-Count`, selected atomically with the artifact lease. Streaming holds the lease until the response ends or the client disconnects. The accepted endpoints return `404` until a comparable checkpoint exists: @@ -341,6 +347,8 @@ validation for a change; some tests require Anvil or libslirp. - [`docs/threat-model/README.md`](docs/threat-model/README.md) — trust boundaries, in-scope and out-of-scope threats. - [`docs/recovery/README.md`](docs/recovery/README.md) — automatic recovery, TLA+ formal verification, design history. - [`docs/recovery/cockroach.md`](docs/recovery/cockroach.md) — manual rebuild after lost state or a sequencer bug. +- [Application history and replay](docs/protocol/application-history.md) — progress, history identity, and replica bootstrap/resume. +- [Snapshots](docs/snapshots/README.md) — engine checkpoints, lifecycle, accepted comparison, and wallet encoding. - [`docs/watchdog/getting-started.md`](docs/watchdog/getting-started.md) — step-by-step: run the watchdog with a local sequencer. - [`docs/watchdog/operator-deployment.md`](docs/watchdog/operator-deployment.md) — watchdog on live L1 (Sepolia staging, mainnet production). - [`docs/watchdog/README.md`](docs/watchdog/README.md) — watchdog architecture, modules, and test commands. diff --git a/docs/plans/2026-07-coordination-tracks.md b/docs/plans/2026-07-coordination-tracks.md index 85222678..b4781d77 100644 --- a/docs/plans/2026-07-coordination-tracks.md +++ b/docs/plans/2026-07-coordination-tracks.md @@ -14,17 +14,17 @@ freely at this stage — no backward-compatibility constraints. |---|-------|-------|--------| | 1 | WS context fields + L1 provenance (PR #26) | Stephen | **done** — merged to main | | 2 | Restore `docs/review/` ledger + this plan | us | **done** | -| 3 | Feed & replay protocol redesign | us (design) → us/Stephen (impl) | **implemented** — canonical application history, snapshot restore archives, mandatory WS claims, typed refusals, and SDK cutover; [remaining integration gates](2026-07-track3-feed-replay-design.md#5-acceptance-evidence-and-remaining-work) | +| 3 | Feed & replay protocol redesign | us (design) → us/Stephen (impl) | **implemented** — canonical application history, snapshot restore archives, mandatory WS claims, typed refusals, and SDK cutover; [remaining integration gates](2026-07-track3-feed-replay-design.md#remaining-integration-gates) | | 4 | Storage decode policy | us | **done** — fail-loud for contract-impossible values; the named `saturating_query_bound` only where clamping preserves the predicate (policy lives in `storage/convert.rs` + the invariants check policy) | | 5 | Fee exponentiation LUT | us | **deferred** — decided exact-floor if built (the table *is* the spec, algorithm-free; replay continuity across the upgrade explicitly not preserved); a separate pending design decision may make log-space fees defunct — revisit after syncing with Bart | -| 6 | Dump / `Application` API redesign | us + Bart | **revised interface implemented** — [Application contract](../protocol/application-contract.md); native bridge conformance is a separate integration branch | +| 6 | Dump / `Application` API redesign | us + Bart | **interface and reference C binding implemented** — [Application contract](../protocol/application-contract.md); native-engine integration gates remain | | 7 | LLM context-engineering review | us | **done** — skills/agents/settings homed in-tree; the docs-practice rules live in AGENTS.md | | 8 | Runtime ownership and terminal stop | us | **done** — owned by the [authority-boundary ADR](2026-08-authority-boundary-adr.md) | **Current campaign order:** -1. Validate Track 6 against the reference C bridge, then the private DEX engine when shared. -2. Exercise native-engine snapshot bootstrap and remeasure feed latency in the representative environment. +1. Validate snapshot-to-live replica bootstrap through the reference C bridge, then the private DEX engine when shared. +2. Remeasure feed latency in the representative environment. 3. Track 5 (fee LUT) only after the log-space-fees decision. Full restore archives now support file and directory application prefixes. @@ -32,25 +32,15 @@ Additional snapshot retention or transport mechanisms require a measured consume ## Track 3 — Feed & replay protocol redesign -Infrastructure subscribers download an application-defined snapshot over HTTP, -restore their application, and use one WS stream for both canonical backlog and -live inputs. The [design](2026-07-track3-feed-replay-design.md) owns the history -claims, typed refusals, resource bounds, and fresh-snapshot recovery workflow. -Raw `/inputs` and separate HTTP transaction replay are outside this feature. -The watchdog retains its independent trusted-state/L1 comparison workflow. +The current [history contract](../protocol/application-history.md) owns replica +bootstrap, history identity, replay, and recovery boundaries. The +[API contract](../../README.md#api) owns wire behavior. The wallet's cold replica +and canonical recovery/watchdog gates have a +[validation record](../review/2026-09-16-track3-validation.md). -The implemented path uses one current `application_inputs` projection for catch-up -and egress. Snapshot headers identify the same leased artifact being downloaded; -WS claims name an era, generation, and inclusive next-input count. A valid -available backlog is replayable without a total catch-up cap, with bounded pages, -queues, and subscribers. Recovery refuses old claims before delivering inputs. - -The former physical replay cursor and sparse attribution design are superseded -by the [application-history design](application-history.md). The wallet's cold -replica and canonical recovery/watchdog gates have a -[validation record](../review/2026-09-16-track3-validation.md). Native-engine -bootstrap and representative latency measurements remain integration gates; -no additional protocol layer is assumed for them. +Remaining work is native-engine integration and representative deployment +latency, tracked in the [integration plan](2026-07-track3-feed-replay-design.md). +Additional transport or retention mechanisms require a measured consumer need. ## Track 5 — Fee exponentiation LUT (deferred) @@ -70,23 +60,15 @@ until the pending log-space-fees decision lands (with Bart). ## Track 6 — Dump / `Application` API redesign -The accepted boundary keeps checkpoint creation, restore, disposal, and a pure -path to canonical comparison bytes. Creation takes `&mut self`, allowing an -adapter to flush or replace backing mappings while preserving logical state. -Checkpoints are durable before SQLite references them, immutable afterward, -and independently restorable even after source deletion. The application -prefix may be a file or directory. - -The engine owns count/clock progress and reports it by value. Successful apply -hooks advance it; the shared boundary verifies the exact successor. Keep -`Send`, remove unused `Clone + Sync`, and place canonical inspection on its -actual consumer. See the [Application contract](../protocol/application-contract.md) -for migration and the [review ledger](../review/2026-09-09-application-lane-dex-review.md) -for the accepted simplifications. - -The [July proposal](2026-07-track6-dump-api-design.md) is superseded. CoW, -flush/reopen sequencing, and working-image management belong inside an engine -adapter. Additional public primitives or asynchronous checkpoint scheduling -need a measured requirement. The DEX's private scheduler and bridge have not -been shared; conformance of the reference C bridge cannot establish theirs. -A watchdog comparison against the canonical DEX state drive is separate work. +The [Application contract](../protocol/application-contract.md) owns execution, +engine progress, and checkpoint semantics. The [C binding guide](../protocol/c-application-binding.md) +maps that contract to native engines; its reference conformance suite is +implemented. End-to-end native snapshot-to-live bootstrap remains an integration +gate, alongside the private DEX engine when available. Reference bridge +conformance cannot establish private-engine correctness. + +The [July proposal](2026-07-track6-dump-api-design.md) is historical; the +[September review](../review/2026-09-09-application-lane-dex-review.md) records the +accepted simplifications. Additional public checkpoint primitives or asynchronous +scheduling need a measured requirement. Watchdog extraction from the DEX's +canonical state drive remains separate work. diff --git a/docs/plans/2026-07-track3-feed-replay-design.md b/docs/plans/2026-07-track3-feed-replay-design.md index f4546f49..6a2700d7 100644 --- a/docs/plans/2026-07-track3-feed-replay-design.md +++ b/docs/plans/2026-07-track3-feed-replay-design.md @@ -1,135 +1,33 @@ -# Feed and Replay Protocol (Track 3) +# Feed and replay integration (Track 3) -**Status: implemented in the current application-history redesign.** -The former physical-rowid feed and sparse execution mapping are superseded. -The [README](../../README.md) owns the wire contract; the -[application-history design](application-history.md) owns storage and recovery -boundaries. This document records the consumer workflow and remaining gates. +The application-history protocol is implemented. Its current contracts live in: -## 1. Consumer workflow +- [Application history and replay](../protocol/application-history.md): coordinates, + storage/recovery boundaries, snapshot bootstrap, and consumer resume. +- [README API](../../README.md#api): routes, wire messages, refusal codes, and limits. +- [Snapshot lifecycle](../snapshots/lifecycle.md): durable artifacts, accepted + comparisons, recovery exports, and leases. -1. Download `GET /latest_snapshot` using the SDK. Its tar body contains the - complete immutable restore artifact (`info.toml` and the opaque `state` - file or directory). -2. Restore the application and verify its executed-input count against - `X-Executed-Input-Count`. Keep the matching `X-History-Era` and - `X-Recovery-Generation` headers with those bytes and the restored state. -3. Subscribe with that `HistoryClaim`: mandatory `era_id`, - `recovery_generation`, and `next_input` query fields. -4. Apply each entry whose `offset` equals the application's current count. - Successful execution advances the count by one. Persist identity with the - replicated state before using it for a later resume. -5. After an ordinary disconnect, reconnect with the saved identity and actual - next-input count. On an era or generation refusal, discard the incompatible - replica and bootstrap from a current snapshot. +## Remaining integration gates -A fresh identity lookup cannot authorize old state. The SDK requires an explicit -claim on every subscription; it does not silently change identity on reconnect. -There is no separate history-version endpoint. +1. Validate the native reference adapter's snapshot-to-live replica workflow; + repeat against the private DEX bridge when available. Reference-engine + conformance does not establish private-engine correctness. +2. Measure submit-to-matching-WS-event latency in the representative deployment, + including checkpoint creation and L1 reconciliation under the supported load. -Snapshot selection, its count, history version, and GC lease share one storage -transaction. The lease lasts through response completion or disconnect. A -recovery between snapshot acquisition and subscription is handled by refusing -its old claim, without blocking history advancement during transfer. - -## 2. Coordinates and storage - -- `safe_inputs.safe_input_index` names a source L1 InputBox event. It includes - scheduler batch envelopes and direct application inputs. -- `application_inputs.offset` is an `ExecutedInputCount`: an application at - count `N` consumes entry `N` next. Every row has an offset, owner frame, and - exactly one user-op or source-L1 reference. Batch envelopes never appear. -- The immutable era baseline supplies the unavailable application prefix - `K` and accounted L1 block. Current entries occupy `[K, H)`, where `H` is - the next application count. -- Standard recovery deletes an invalidated projection suffix and advances - its generation atomically. Replacement inputs reuse those canonical - offsets. Raw L1 inputs, batches, frames, and user ops retain their source - evidence; the invalidated flattened sequence is not separately retained. -- Rebuild creates a new UUIDv4 era and a complete baseline. It does not insert - padding inputs or preserve a physical replay cursor. - -Catch-up and egress use the same named entry and coherent canonical-page -reader. Bounds, identity, and rows are read in one SQLite transaction. Missing -interior rows and invalid payload context fail loudly. Empty requests at the -head do not convert the exclusive boundary back into a SQL row coordinate. - -The latest valid frame's `safe_block`, bounded below by the era's L1 baseline, -accounts for the complete L1 prefix. No separate mutable processed-input cursor -is needed. Snapshot application count and L1 accounting are different facts; -see the application-history design for recovery's terminal drain and sparse -checkpoint availability. - -## 3. Subscription admission - -Validate history identity before position, using one coherent `(era, -generation, K, H)` read: - -| Condition | HTTP 409 policy code | Consumer action | -|---|---|---| -| Era differs | `ERA_CHANGED` | Bootstrap from a current snapshot. | -| Generation differs | `STALE_GENERATION` | Bootstrap from a current snapshot. | -| `N < K` | `HISTORY_UNAVAILABLE`, with `available_from` | Bootstrap from an available snapshot. | -| `N > H` | `AHEAD_OF_HEAD`, with `head` | Correct the invalid claim. | -| `K <= N <= H` | Upgrade to WebSocket | Replay inclusively from `N`, then follow the tip. | - -Refusals precede the upgrade and all input delivery. The JSON body is also -carried in `X-History-Error`: WebSocket libraries may stop reading a refused -handshake at its headers before the body arrives. Missing or malformed required -query fields receive HTTP 400. - -A successful stream carries the existing tagged user-op/direct-input messages -with canonical offsets and persisted context. The mandatory admission claim -binds the session identity; there is no hello frame or per-event generation. -Recovery changes history only across a process boundary, after existing -subscriptions have ended. No generation bus or farewell guarantee is needed. - -`N == H` waits normally. Every valid available backlog is replayable: there is -no total 50,000-event cap. Page size, send queue, subscriber count, and inbound -message limits remain bounded independently of backlog depth. The same durable -query handles backlog and live delivery, avoiding a separate handoff cursor. - -## 4. Snapshot and watchdog boundaries - -`/latest_snapshot` is a replica restore archive. `/finalized_state` remains the -watchdog's application comparison bytes, with its inclusion-block metadata -route. `/finalized_snapshot` exports an accepted recovery artifact and a derived -`checkpoint.toml` receipt. These are operator-infrastructure routes. - -The watchdog starts from trusted state and independently consumes L1; snapshot -bootstrap for a tip replica does not replace that trust boundary. Finalized -comparison/export is available only at a supported accepted checkpoint, not at -an invented intra-frame or arbitrary execution position. - -## 5. Acceptance evidence and remaining work - -The implementation tests cover inclusive pages and source context, exclusion -of envelopes, nonzero rebuild bases, actual suffix invalidation and replacement, -coherent SQLite snapshots during a second writer's recovery, counts beyond the -largest SQL row, and loud interior-gap detection. Feed/API/SDK tests cover -mandatory claims, typed refusals, exact-head waiting and live delivery, -50,001-entry history with bounded pages, ordinary resume, subscriber limits, -terminal storage faults, and cancelled preparation retaining process ownership. -Snapshot integration tests own artifact/header association and restore proof. -The Anvil recovery/WS gate also exercises process restart, generation refusal, -and re-drained direct replay at a reused offset. - -The cold-replica E2E restores a nonempty HTTP archive, checks it against a -genesis-fed replica, and holds catch-up behind a barrier while new writes commit. -It checks whole-state, count, and clock agreement through live direct inputs -and user operations, then exercises real stale recovery, claim refusal, and -fresh bootstrap. Canonical-machine gates cover genesis, ordinary execution, -stale recovery, and database reconstruction from an exported checkpoint. The [validation record](../review/2026-09-16-track3-validation.md) records the -pinned environment and local latency measurements. +wallet's nonempty cold bootstrap, concurrent replay/live delivery, recovery and +rebootstrap, canonical-machine gates, and local latency measurements. Those +results do not replace the consumer/environment gates above. -Remaining integration gates are concrete consumers and environments: +## Revisit only with a consumer need -- Validate the native reference adapter and, when available, the private DEX - bridge against the application contract and this bootstrap workflow. -- Remeasure submit-to-matching-WS-event latency on the representative deployment. +- Resumable snapshot transfer: when artifact size makes interrupted downloads costly. +- Retained client checkpoints: when full rebootstrap cost matters. +- Archival HTTP replay or raw L1 feeds: for an identified consumer. +- Session fencing: if history can mutate within an admitted process or multiple + local writers become supported. -Revisit resumable snapshot transfer only when artifact size requires it; -retained client checkpoints only when full rebootstrap cost matters; archival -HTTP replay only for an identified consumer. Revisit session fencing if history -can mutate within an admitted process or multiple writers become supported. +These are triggers for design work, not promised APIs. The +[coordination plan](2026-07-coordination-tracks.md) owns cross-track priorities. diff --git a/docs/plans/2026-07-track6-dump-api-design.md b/docs/plans/2026-07-track6-dump-api-design.md index 93c25c1d..528ce0a7 100644 --- a/docs/plans/2026-07-track6-dump-api-design.md +++ b/docs/plans/2026-07-track6-dump-api-design.md @@ -1,4 +1,4 @@ -# Dump / `Application` API — Design Draft (Track 6) +# Historical dump / `Application` proposal (Track 6) **Status: superseded by the 2026-09-09 design decision.** Preserved as the original proposal; its public clone/flush machinery was not adopted. The diff --git a/docs/plans/2026-08-authority-boundary-adr.md b/docs/plans/2026-08-authority-boundary-adr.md index 72b78965..305e73f4 100644 --- a/docs/plans/2026-08-authority-boundary-adr.md +++ b/docs/plans/2026-08-authority-boundary-adr.md @@ -170,8 +170,8 @@ standard-recovery transaction iff it invalidates at least one valid batch; a clean restart changes neither. The pair is an equality/discontinuity token, not an ordered counter. Snapshot headers and mandatory WS claims expose these coordinates. Every application row has its pre-execution count; recovery replaces only the current -suffix. See the [history contract](application-history.md) and -[Track 3 handoff](2026-07-track3-feed-replay-design.md). +suffix. See the [history contract](../protocol/application-history.md) and +[remaining integration gates](2026-07-track3-feed-replay-design.md). ## Performance posture diff --git a/docs/plans/application-history.md b/docs/plans/application-history.md deleted file mode 100644 index e7b935f6..00000000 --- a/docs/plans/application-history.md +++ /dev/null @@ -1,92 +0,0 @@ -# Application history and checkpoints - -The sequencer keeps L1 observations, application ordering, and batch acceptance -as separate durable facts. L1 inputs include batch envelopes; application -history contains only included user operations and external direct inputs. -Source references provide provenance without defining a mapping between the -two timelines. - -## History - -`application_inputs` is the current application sequence. Its primary key is -the input's pre-execution `ExecutedInputCount`; each row belongs to a local -batch/frame and references either its user operation or its source L1 input. -Every row executes. Payloads remain in their source tables. - -The latest surviving frame's `safe_block` records complete L1 accounting. -Reconciliation covers the whole newly safe interval before committing its new -frame and application inputs. Full-block ingestion and indivisible range -reconciliation make a separate mutable processing cursor unnecessary. Empty -intervals and intervals containing only batch envelopes advance this boundary -without adding application inputs. - -Recovery invalidates a batch suffix, removes its current application rows, -advances the history generation, and opens the replacement tip atomically. -Replacement inputs reuse suffix offsets under the new generation. Original -L1, batch, frame, and user-operation records remain available for diagnostics. - -## Era baseline - -Setup registers a complete baseline after its artifact is durable: application -count `K`, accounted L1 stop block `C`, starting batch nonce, and history identity. -The recovered L1 prefix through `C` is opaque to ordinary operation. Both direct -ordering and accepted-batch scanning begin after it. The baseline metadata -survives root invalidation and artifact garbage collection. - -The recovery fold drains queued directs through `C`, including young directs -that the canonical scheduler has not executed yet. Its output is a restart -baseline, without a claim that its bytes equal canonical state at block `C`. -Genesis supplies the trusted block-zero comparison state. - -## Snapshots and acceptance - -The lane creates a durable snapshot at every batch close. Snapshot registration -and batch sealing commit together. Snapshots reference immutable local batch -identities; a nonce can be reused by recovery. The baseline is a separate -snapshot origin. - -Acceptance is derived from complete safe L1 observations, the scheduler's -acceptance rules, and byte identity with the local sealed batch. An accepted -batch confirms existing application history and adds no replay entry. - -Checkpoint selection uses these facts directly: - -- Restart and replica bootstrap use the newest surviving batch snapshot, or - the baseline. -- Recovery requires a retained accepted snapshot, or the baseline before the - first post-baseline acceptance. -- The watchdog compares an accepted checkpoint at the end of its L1 inclusion - block. Per-batch snapshots make the latest accepted batch's artifact available. - -Select the required accepted batch before loading its snapshot: a missing -required artifact is an invariant violation, never permission to choose an -older checkpoint. Divergence blocks publication of a newly derived comparison. - -There is no snapshot promotion mutation. Retention keeps the newest accepted -snapshot (or baseline), all valid snapshots beyond the accepted frontier, and -leased artifacts. The baseline bytes can be retired once an accepted artifact -provides the recovery fallback. Artifact creation precedes DB publication; -DB retirement precedes filesystem deletion. - -A portable accepted checkpoint includes the application artifact and coherent -sequencer metadata identifying its canonical comparison point and resume nonce. -Acceptance metadata is derived at export; application artifacts stay immutable. -Sparse snapshot creation and intra-block watchdog checkpoints are separate work. - -## Replay and egress - -Restart and egress share application-only pages beginning at an inclusive input -count. Snapshot bootstrap uses HTTP; one WS stream replays available history -then follows the tip. Subscription claims include era, generation, and next -input count. Wrong identity or unavailable history requires bootstrap; a claim -at the head waits and one beyond it fails. Pages and queues are bounded, while -total replay has no arbitrary catch-up cap. - -## Validation boundaries - -Exercise prefix exclusion for previously rejected future-nonce batches; -baseline-only restart and repeated root invalidation; envelopes-only frame -advancement; acceptance observed during downtime; empty accepted batches; -snapshot retirement with active leases; atomic suffix replacement; and cold -replica restore followed by canonical replay. The recovery models constrain -admission and batch safety, not the concrete snapshot/GC implementation. diff --git a/docs/protocol/application-contract.md b/docs/protocol/application-contract.md index 40f0c8f1..536a69ab 100644 --- a/docs/protocol/application-contract.md +++ b/docs/protocol/application-contract.md @@ -34,6 +34,8 @@ Before an apply hook, the boundary computes the checked expected successor. After `Ok`, it asserts that the engine reports exactly that successor and returns the input's pre-execution count as its history offset. An engine may use `ApplicationProgress::advance` or implement the same transition natively. +Adapters report the engine-owned progress rather than maintaining a separate +count or clock mirror. Overflow fails before the hook runs. An error defines no successor: callers terminate the execution path and discard the instance, without attempting to roll back or inspect partially updated state. @@ -56,6 +58,9 @@ live execution, canonical execution, and replay. State changes happen through apply hooks; dump creation may change backing resources but preserves logical state. +Wrapping an existing engine must preserve its transaction encoding, +rejection/inclusion semantics, and canonical state bytes. + `Application: Send + Sized` permits moving the engine to the lane's blocking worker. It requires neither `Sync` nor `Clone`: the lane owns one mutable engine, and an independent state fork is a fallible checkpoint/restore @@ -117,7 +122,8 @@ execution offsets, checked during catch-up. HTTP snapshot metadata and mandatory WS claims carry the history version and this count. Both restart and subscriber replay read the same current application -sequence; see the [API contract](../../README.md). +sequence. The [history guide](application-history.md) owns coordinates and +replica bootstrap; the [API contract](../../README.md) owns wire shapes. ### 5. Operational capacity for L1 reconciliation @@ -138,8 +144,18 @@ or measured checkpoint latency demonstrates the need. A **recovery checkpoint** contains everything needed to resume the engine. A **canonical comparison file** contains the deterministic state the watchdog compares against the canonical application. They may be the same file; a -machine checkpoint may instead contain a separate app-state projection. The -[format contract](../snapshots/format.md) describes three relevant layouts. +machine checkpoint may instead contain a separate app-state projection. + +| Engine | Recovery checkpoint | Canonical comparison file | +|---|---|---| +| Wallet | SSZ wallet state | The same SSZ file, also returned by canonical inspect | +| Cartesi Machine wrapper | Full multi-file machine state | Deterministic app-state projection stored alongside it | +| Native DEX design | Fixed-memory state `M` plus required resumable metadata | Canonical `M`, matching the designated drive in the canonical machine | + +The DEX row describes an integration requirement, not a verified private +implementation. The [watchdog guide](../watchdog/README.md) owns comparison +transport and support; the [wallet format](../snapshots/format.md) owns its SSZ +representation. - `create_dump(&mut self, prefix)` creates a checkpoint at an absent path, which may become a file or directory. On `Ok`, all files and directory @@ -177,26 +193,3 @@ implementation. `CanonicalState::canonical_snapshot_bytes` is a separate inspection trait required by the shared Rust scheduler's inspection method and canonical harness, not by the native sequencer. Human-readable debugging state also stays on the concrete application. - -## Adapter migration - -1. Remove the capability parameters and mutable progress accessor. Return the - native count/clock pair from `progress()` without a Rust-side mirror. -2. Advance both fields inside each successful native apply transition, - including no-ops. Keep validation pure and map its fatal failures to - `AppError`, with expected rejection as `Ok(ValidationOutcome::Reject(...))`. -3. Change checkpoint creation to `&mut self` and establish durable, immutable - checkpoints with independent restores. Preserve existing canonical bytes. -4. Implement `CanonicalState` only where canonical inspection needs it. - Remove any `Clone` or `Sync` added solely to satisfy the old host bounds; - justify `Send` against the native engine's ownership contract. -5. Return the stable payload bound from `max_method_payload_bytes()` rather - than an associated constant. Keep checkpoint artifacts self-contained so - recursive filesystem deletion disposes of them without an application hook. - -Changing these Rust interfaces preserves transaction encoding, expected -rejection semantics, snapshot bytes, scheduler ordering, and the database -schema. Fatal `AppError` propagation is an intentional exception: validation -and execution failures discard the engine rather than becoming a rejection -or an included no-op. The host's terminal-versus-retryable classification -still applies. diff --git a/docs/protocol/application-history.md b/docs/protocol/application-history.md new file mode 100644 index 00000000..20232418 --- /dev/null +++ b/docs/protocol/application-history.md @@ -0,0 +1,150 @@ +# Application history and replay + +Application history is the sequencer's current execution order: included user +operations and external direct inputs. It contains an optimistic suffix that +recovery may replace. A stable offset identifies an input only together with +its history version; receiving it does not establish L1 acceptance. + +This document owns history coordinates, recovery boundaries, and replica +bootstrap. The [Application contract](application-contract.md) owns execution +and progress; the [snapshot lifecycle](../snapshots/lifecycle.md) owns artifact +publication, selection, and retention; the [README API](../../README.md#api) +owns routes, wire fields, and refusal codes. + +## Progress, ordering, and acceptance + +These facts answer different questions: + +| Fact | Meaning | +|---|---| +| `ExecutedInputCount` | Number of application inputs already executed. At count `N`, entry `N` executes next. | +| `ApplicationProgress.last_executed_safe_block` | Maximum clock carried by an executed input: frame safe block for user ops, inclusion block for directs. | +| Latest surviving frame's `safe_block` | Complete L1 interval accounted for by local ordering, bounded below by the era baseline. | +| `safe_accepted_batches` | Safe L1 landings accepted by the scheduler rules and matched to local sealed bytes. | + +A frame can account for an empty interval or only batch envelopes, advancing +L1 accounting without executing an application input. An accepted batch can +confirm existing execution without adding an input. Empty batches can share +an application count. Neither the count nor the app clock substitutes for the +L1 accounting boundary or acceptance facts. + +`safe_inputs` retains all InputBox observations, including batch envelopes. +`application_inputs` contains only the current ordered application sequence; +each row has a mandatory pre-execution offset, an owning batch/frame, and +exactly one reference to a user op or external direct input. Payloads remain +in those source tables. Included business failures and malformed-direct no-ops +advance the count; validation rejections and envelopes do not. + +Restart and egress read this same sequence. Replay executes stored valid +user ops with their recorded fee and frame clock, without revalidation; +directs use their original inclusion block. Timestamps and transaction hashes +are provenance, not extra application-transition inputs. The +[execution contract](application-contract.md#the-execution-methods) defines the +shared execution boundary. + +## Identity and available history + +A `HistoryClaim` combines: + +- **Era**: a UUIDv4 created by setup/rebuild, identifying one local history. +- **Recovery generation**: a revision within that era, advanced atomically + whenever automatic recovery invalidates at least one valid batch. +- **Next input**: the application's actual executed-input count. + +If the era baseline count is `K` and the current head is `H`, available entries +occupy `[K, H)`. A claim at `H` waits for future entries; a claim below `K` or +above `H` is refused. Identity is checked before position. Equal counts cannot +authorize resuming a different era or generation, even if the consumer believes +its state precedes the replaced suffix. + +Automatic recovery invalidates a batch suffix, removes its current application +rows, advances the generation, and opens the replacement Tip in one transaction. +Replacement inputs reuse suffix offsets under the new generation. Original L1, +batch, frame, and user-op source records remain; the invalidated flattened +sequence is not separately retained. A repair that invalidates nothing leaves +the generation unchanged. The [recovery guide](../recovery/README.md) owns +repair selection and guards. + +### Era baseline + +Setup publishes a complete baseline only after its artifact is durable: +application count `K`, accounted L1 stop block `C`, starting batch nonce, and +history identity. Ordinary direct ordering and accepted-batch scanning begin +after `C`. The baseline's metadata survives root invalidation and artifact GC; +no padding inputs or separate mutable processing cursor represent its prefix. + +Cockroach recovery drains queued directs through `C`, including young directs +that the canonical scheduler has not executed yet. Its output is a stable +restart baseline; its bytes need not equal canonical state at block `C`. +A later accepted batch snapshot supplies the first comparison checkpoint in that era. +Genesis supplies the trusted block-zero comparison state. The +[rebuild guide](../recovery/cockroach.md) owns checkpoint requirements and the +fixed stopping boundary. + +## Three consumers of checkpoints + +| Consumer | Starting point and continuation | +|---|---| +| Native restart | Load the newest surviving batch snapshot, or baseline, check the engine's count against its row, then replay current application inputs. | +| Sequencer replica | Download `/latest_snapshot`, restore its application state, and subscribe using the matching history claim. This follows optimistic execution. | +| Watchdog | Start from independently trusted canonical machine state and replay L1. Compare at the sequencer's accepted checkpoint; the replica feed does not establish independent trust. | + +A batch-close snapshot is identified by its local batch identity, not just its +count or nonce. Recovery can reuse a nonce and empty batches can repeat a count. +Acceptance derives the comparison point without modifying the artifact. +Selection must first choose the required accepted batch and then require its +snapshot; a missing one cannot justify falling back to an older comparison. +The [snapshot lifecycle](../snapshots/lifecycle.md#acceptance-and-comparison) +explains block-boundary comparison and rollback retention. The +[watchdog guide](../watchdog/README.md) explains independent verification. + +## Replica bootstrap and resume + +1. Download `GET /latest_snapshot`. The tar archive contains `info.toml` and + the complete opaque application `state` file or directory. +2. Retain its `X-History-Era`, `X-Recovery-Generation`, and + `X-Executed-Input-Count` headers with those bytes. Restore the application and + verify its count against that header. The archive alone does not carry the + complete subscription claim. +3. Subscribe with the matching era, generation, and next-input count. Snapshot + selection, headers, and lease share one transaction; recovery during the + download can still invalidate the claim before subscription. Rebootstrap + if the server refuses that old identity. +4. Require each entry's offset to equal the application's current count, then + execute it through the shared execution boundary. Successful application + advances the count by one. Persist the history identity with the replica's + state so that a later resume cannot combine different histories. +5. After an ordinary disconnect, reconnect with that saved identity and the + actual count. A history mismatch or unavailable prefix requires a current + snapshot. A count ahead of the server's head is an invalid claim to correct. + +The [Rust SDK](../../sdk/rust-client/src/lib.rs) returns a `HistoryClaim` with +its snapshot response and requires an explicit claim for subscriptions. The +consumer owns restore, persistence, and reconnect. Fetching fresh identity +metadata cannot authorize old application state. + +One durable page reader handles both backlog and live delivery, so there is no +separate cursor to switch at the live boundary. Each page reads identity, +bounds, and rows in one SQLite transaction. Interior gaps and invalid source +context fail loudly. Page size and send queues bound memory; available backlog +has no total replay cap. The [API contract](../../README.md#api) owns exact +resource limits and handshake errors. + +History replacement happens across a process boundary, after existing +subscriptions end. A session's mandatory claim therefore binds all its events; +there is no per-event generation or guaranteed farewell message. Revisit this +assumption if history can change within an admitted process or multiple local +writers become supported. + +## Code and validation map + +| Boundary | Code and tests | +|---|---| +| Coordinates and claim ordering | [`sequencer-core/src/history.rs`](../../sequencer-core/src/history.rs) — identity before position, inclusive head, checked counts. | +| Coherent application pages | [`storage/egress/canonical.rs`](../../sequencer/src/storage/egress/canonical.rs) and its tests — source context, nonzero baselines, replacement offsets, concurrent recovery, gaps and SQL limits. | +| Replay followed by live delivery | [`l2_tx_feed`](../../sequencer/src/egress/l2_tx_feed/) — bounded deep replay, identity refusals, shutdown and persistent faults; [`catch_up.rs`](../../sequencer/src/ingress/inclusion_lane/catch_up.rs) for native replay. | +| Artifact and claim association | [`snapshot_endpoints.rs`](../../sequencer/src/integration_tests/snapshot_endpoints.rs) — headers, restore, archive contents, and lease lifetime. | + +The [integration validation record](../review/2026-09-16-track3-validation.md) +records wallet replica and canonical-machine evidence. Remaining consumer and +deployment gates belong to the [Track 3 plan](../plans/2026-07-track3-feed-replay-design.md). diff --git a/docs/protocol/scheduler-semantics.md b/docs/protocol/scheduler-semantics.md index 4d1550a0..8b5cb815 100644 --- a/docs/protocol/scheduler-semantics.md +++ b/docs/protocol/scheduler-semantics.md @@ -181,7 +181,7 @@ history is no longer locally available. The durable `application_inputs` sequence uses these same offsets. HTTP snapshots carry the history version and application count; WS subscriptions -must claim both before inclusive replay. See the [history contract](../plans/application-history.md). +must claim both before inclusive replay. See the [history contract](application-history.md). --- diff --git a/docs/recovery/cockroach.md b/docs/recovery/cockroach.md index 0016e269..ff588f73 100644 --- a/docs/recovery/cockroach.md +++ b/docs/recovery/cockroach.md @@ -123,7 +123,7 @@ orphan artifact. Identity pinning and raw L1 ingestion may survive an incomplete attempt, but the lock and setup admission prevent serving a partial baseline. `C` remains the fallback reconciliation boundary if standard recovery invalidates -the root. The [history contract](../plans/application-history.md#era-baseline) +the root. The [history contract](../protocol/application-history.md#era-baseline) owns these immutable coordinates; [snapshot lifecycle](../snapshots/lifecycle.md) owns restore selection, rollback-safe retention, and eventual baseline disposal. diff --git a/docs/review/2026-09-09-application-lane-dex-review.md b/docs/review/2026-09-09-application-lane-dex-review.md index 9009cd31..a4bfa345 100644 --- a/docs/review/2026-09-09-application-lane-dex-review.md +++ b/docs/review/2026-09-09-application-lane-dex-review.md @@ -27,7 +27,7 @@ cherry-picked into the main implementation. The private DEX scheduler and native engine are still unavailable. Reference bridge tests verify the proposed seam, not private engine conformance. DEX -conformance and the [Track 3 history API](../plans/2026-07-track3-feed-replay-design.md#7-ordered-implementation-handoff) +conformance and the [Track 3 history API](../plans/2026-07-track3-feed-replay-design.md) remain follow-ups. This review establishes reference integration coverage; it does not establish that the Application surface is production-proven. diff --git a/docs/review/register.md b/docs/review/register.md index fadd3892..8942aa8a 100644 --- a/docs/review/register.md +++ b/docs/review/register.md @@ -53,7 +53,7 @@ remaining dated ledgers stay valid. 7. **Closed** (2026-09-16): mandatory era/generation/application-count claims reject resume across recovery; snapshot headers provide cold-bootstrap coordinates. Current application suffix replacement is atomic with the - generation bump. See the [Track 3 contract](../plans/2026-07-track3-feed-replay-design.md). + generation bump. See the [history contract](../protocol/application-history.md). 8. **Fee-determinism contract under-specified** — the LSB-first floor-after-each-multiply order is implemented but not stated as contract (`sequencer-core/src/fee.rs`). Load-bearing for the C++ scheduler port; @@ -291,7 +291,7 @@ Each entry: the decision, its reason, and where the reasoning now lives. versioned claims replace the mixed replay log and sparse mapping. Acceptance facts select immutable per-batch snapshots without promotion or restamping; per-batch cadence and end-of-block watchdog comparison remain. The complete - model lives in [application history](../plans/application-history.md), I5–I11, + model lives in [application history](../protocol/application-history.md), I5–I11, I18/I20, and the snapshot lifecycle. - **No architectural restructure** (2026-06-10): one file per writer role, @@ -705,5 +705,5 @@ for `2026-06-10-correctness-review.md`, `2026-06-10-simplification.md`, | 2026-09-07 | PR #28 premise review and maintainer-approved simplification | Ordered recovery replaces the phase driver; diagnosed terminal runtime faults abort immediately; ordinary shutdown and snapshot leases remain; reader drain race and duplicate offset check fixed | Current ADR and recovery design; finding 33 and settled decisions above. Validation: 692 host tests, seven targeted restart/outage E2Es, workspace check, strict Clippy, formatting, and admission TLC passed. The broader stale-batch recovery E2E reached its watchdog comparison but was blocked by the host Lua emulator 0.21 loading the pinned 0.20 image (archive version mismatch); no protocol pin was changed. | | 2026-09-09 | Application, inclusion lane, and public DEX integration branch | Native progress ownership, typed validation failures, mutable independent checkpoints, optional canonical inspection, and lane bookkeeping simplified; ingress CORS and Lua 5.4 parity restored. Reference C bridge port kept separate. | [Application/lane review](2026-09-09-application-lane-dex-review.md); current Application and snapshot contracts. Workspace check, strict Clippy, 697 host tests, and 62 watchdog tests passed; private DEX conformance remains unverified. | | 2026-09-11 | Reference C bridge port and review boundary | Keep the current Application contract, runtime payload bound, paired progress, filesystem-owned checkpoint disposal, and host failure fixes together. Engine-dependent API refinements and external-engine conformance remain follow-ups. | Settled decisions and owed tests above; [C binding guide](../protocol/c-application-binding.md), Application contract, and snapshot lifecycle. | -| 2026-09-16 | Application-history and Track 3 implementation, with independent storage/recovery/snapshot review | Replaced mixed replay and sparse attribution with current application inputs; complete atomic baselines; acceptance-derived immutable snapshots; HTTP restore/recovery archives and mandatory WS claims. Review narrowed equal-block recovery to empty genesis to avoid losing same-block pending directs. | [History design](../plans/application-history.md), current invariants/API/snapshot/recovery docs. Validation: 693 workspace tests, strict Clippy, formatting, 62 watchdog tests, admission TLC (155 distinct states), and the Anvil recovery/old-claim refusal gate passed. Full canonical watchdog comparison remains blocked by the local emulator 0.21 vs repository 0.20 environment; no pin changed. | +| 2026-09-16 | Application-history and Track 3 implementation, with independent storage/recovery/snapshot review | Replaced mixed replay and sparse attribution with current application inputs; complete atomic baselines; acceptance-derived immutable snapshots; HTTP restore/recovery archives and mandatory WS claims. Review narrowed equal-block recovery to empty genesis to avoid losing same-block pending directs. | [History design](../protocol/application-history.md), current invariants/API/snapshot/recovery docs. Validation: 693 workspace tests, strict Clippy, formatting, 62 watchdog tests, admission TLC (155 distinct states), and the Anvil recovery/old-claim refusal gate passed. Full canonical watchdog comparison remains blocked by the local emulator 0.21 vs repository 0.20 environment; no pin changed. | | 2026-09-16 | Track 3 integration validation | Nonempty HTTP cold replica, concurrent backlog/live consumption, stale recovery/rebootstrap, and four real canonical-machine gates pass under emulator 0.20. Tooling fixes preserve Lua paths and make benchmark fee defaults admissible. | [Validation and latency evidence](2026-09-16-track3-validation.md); native bridge/DEX and representative deployment latency remain separate gates. | diff --git a/docs/snapshots/README.md b/docs/snapshots/README.md index 5b983d3e..63eb0de5 100644 --- a/docs/snapshots/README.md +++ b/docs/snapshots/README.md @@ -5,18 +5,24 @@ known execution boundary. They let the inclusion lane resume with load and replay. A snapshot may contain optimistic state; L1 acceptance determines which artifact can back a canonical comparison or recovery export. -Two documents, split by concern: +Keep three boundaries separate: the engine embeds its executed-input count and +safe-block clock; SQLite associates an artifact with a local batch or era +baseline; safe L1 acceptance selects a comparison checkpoint. Counts and clocks +alone do not establish acceptance. The +[history guide](../protocol/application-history.md) explains these coordinates +and snapshot-plus-replay bootstrap. -- **[`format.md`](format.md)** — the on-disk *format*: the `Application` dump - trait (`from_dump` / `create_dump` / `state_file_in_dump`) and - the toy wallet's SSZ wire encoding. What a dump *is*. - -- **[`lifecycle.md`](lifecycle.md)** — creation at batch close, restart selection, +- [Application contract](../protocol/application-contract.md#6-checkpoint-lifecycle) + — engine dump methods, durability, immutable artifacts, independent restore, + and the canonical comparison file. +- [Wallet format](format.md) — the wallet's layout, deterministic SSZ encoding, + and decode rules. +- [Lifecycle](lifecycle.md) — creation at batch close, restart selection, acceptance-derived comparison checkpoints, recovery exports, retention, download leases, and crash safety. Acceptance is a separate durable fact; artifacts are never promoted or rewritten. -For automatic startup repair, see [standard recovery](../recovery/README.md). +For automatic startup repair, see [automatic recovery](../recovery/README.md). For rebuilding after database loss or a sequencer bug, see [cockroach recovery](../recovery/cockroach.md). The root [README](../../README.md) owns endpoint shapes. diff --git a/docs/snapshots/format.md b/docs/snapshots/format.md index 963a5323..68f409f4 100644 --- a/docs/snapshots/format.md +++ b/docs/snapshots/format.md @@ -1,81 +1,11 @@ -# App Snapshot Format (Wallet Toy App) +# Wallet snapshot format -This document defines the on-disk snapshot format produced by the toy wallet -app in `examples/app-core` via the `Application` trait's dump methods. - -## Scope - -This document covers two things: - -1. The trait shape that any `Application` implementation must satisfy to - participate in snapshot lifecycle (`from_dump`, `create_dump`, - `state_file_in_dump`). -2. The wire format the toy wallet uses to encode its canonical state into - the dump's state file. Checkpoint ownership and durability are defined by -the [Application contract](../protocol/application-contract.md#6-checkpoint-lifecycle). - -It does NOT define when snapshots are triggered, how the inclusion lane -records and selects them, how the HTTP layer serves them, or recovery -interactions. Those are layered above the trait and live in their own -modules. - -## Trait Surface - -```rust -trait Application: Send + Sized { - // ... other methods ... - - fn from_dump(prefix: &Path) -> Result; - fn create_dump(&mut self, prefix: &Path) -> Result<(), AppError>; - fn state_file_in_dump(prefix: &Path) -> PathBuf; -} -``` - -Contract: - -- `prefix` is an opaque app-owned path, which may be a file or directory. - All checkpoint-owned artifacts reside at or below it. The sequencer owns the - enclosing dump directory and its `info.toml`, and disposes of the checkpoint - by removing that directory recursively. No other resource cleanup is needed. -- `create_dump` creates the absent `prefix` and makes the complete checkpoint - durable before returning. It may replace backing resources but preserves - logical state. Later execution cannot alter a checkpoint. Restored engines - are independent of one another and remain usable after source deletion. -- `state_file_in_dump` is a pure function of `prefix`: callers may - compute it without loading the dump or instantiating the Application. - Each impl pins its own layout convention. -- The bytes at `state_file_in_dump(prefix)` are the canonical state — - the bytes a watchdog running an independent canonical machine would - produce for the same logical state through inspect or a designated state - drive. - They must be deterministic: identical logical state must produce - byte-identical files across runs, hosts, and toolchains. -- For implementations whose persistence representation IS the canonical - state (the toy wallet), `create_dump` writes the - same bytes that `state_file_in_dump` names — a single file with no - duplication. For implementations whose persistence is richer than the - canonical state (e.g. a Cartesi Machine wrapping app), `create_dump` - writes the full machine state alongside a separate canonical-state file - under the same prefix. - -The recovery checkpoint and canonical comparison representation differ by app: - -| Engine | Recovery checkpoint | Canonical comparison file | -|---|---|---| -| Toy wallet | SSZ wallet state | The same SSZ file, also returned by canonical inspect | -| Cartesi Machine wrapper | Full multi-file machine state | Deterministic app-state projection stored alongside it | -| Native DEX design | Fixed-memory state `M` plus any required resumable metadata | Canonical `M`, matching the designated drive in the canonical machine | - -The DEX row describes the integration requirement, not a verified private -implementation. The current watchdog reads canonical inspect output; direct -comparison against a canonical drive remains separate watchdog work. A native -adapter does not need to implement the Rust canonical inspection trait to serve -its checkpoint's comparison file. - -CoW is an implementation choice within checkpoint creation and restore. Shared -physical extents are compatible with the contract; shared mutable bytes are -not. The sequencer does not prescribe a filesystem clone primitive or expose -engine flush/reopen steps. +This document owns the wallet's application-state bytes, implemented by +[`wallet_snapshot.rs`](../../examples/app-core/src/wallet_snapshot.rs). +The [Application contract](../protocol/application-contract.md#6-checkpoint-lifecycle) +owns the dump methods, durability, and independent restoration; +[lifecycle.md](lifecycle.md) owns the enclosing artifact, metadata, selection, +and retention. ## Toy Wallet Layout @@ -88,6 +18,13 @@ and its canonical state coincide; one write per `create_dump`. state SSZ-encoded WalletSnapshot bytes ``` +Here `prefix` is the app-owned `state` directory inside the sequencer's dump +directory. Thus the wallet file is `dumps//state/state`. The surrounding +`info.toml` and exported `checkpoint.toml` use the sequencer's metadata format +version; that version does not identify the wallet's SSZ schema. Application +progress is embedded in the wallet bytes. History identity and acceptance +metadata come from the sequencer; see [application history](../protocol/application-history.md). + ## Toy Wallet Wire Format - **Encoding**: SSZ @@ -114,9 +51,6 @@ and its canonical state coincide; one write per `create_dump`. reflects, so it must live in the canonical state bytes (both the bare-metal and canonical-machine sides advance it identically). -`last_executed_safe_block` was added before any environment was deployed; there -is a single, unversioned schema today (see [Versioning](#versioning)). - ### Determinism `WalletApp` stores balances and nonces in `HashMap`s, so iteration order @@ -137,21 +71,16 @@ The decoder rejects: - Malformed SSZ bytes (any decode error from the SSZ library). - A snapshot containing two entries in `balances` with the same address. - A snapshot containing two entries in `nonces` with the same address. +- Zero `executed_input_count` with a nonzero `last_executed_safe_block`. -The duplicate-address checks exist to keep the encoded bytes canonical: -without them, multiple distinct byte sequences could decode to the same -logical state (the second entry would silently overwrite the first), -breaking the property that watchdog-side and sequencer-side bytes are -comparable. +Duplicate checks prevent an entry from silently overwriting another during +restore. The decoder accepts unique entries in any order; encoding the restored +state sorts them. Deterministic emitted bytes do not imply that the decoder +accepts only that ordering. ## Versioning -There is a single, unversioned schema: `WalletSnapshot`. The encoded bytes carry -no leading version tag, and — because there is no backward-compatibility -requirement yet (no long-lived deployment whose dumps a newer binary must read) — -the struct name carries no version suffix either. An earlier draft distinguished -a `V1`/`V2` pair (the `last_executed_safe_block` field was added before any -environment existed); that split was collapsed since no `V1` dumps ever survived. +There is one SSZ schema, `WalletSnapshot`, with no leading version tag. If a future change ever needs to break the wire format against live dumps: @@ -165,23 +94,7 @@ Until then, do not reorder, repurpose, or reinterpret existing fields in place. ## Trust Model -The dump file is part of the sequencer's persistent data directory and -shares its trust boundary. An attacker with write access to the data -directory has already won; no integrity tag, checksum, or HMAC is -included on the snapshot bytes for this reason. Consumers that obtain -the bytes via a less trusted channel (e.g. a future peer-to-peer -distribution mechanism) would need to add an outer integrity layer; the -format itself does not provide one. - -## Out of Scope - -This document deliberately does not define: - -- When the inclusion lane decides to take a snapshot. -- How dumps are registered, selected by acceptance, or - garbage-collected. -- The on-the-wire archive format for streaming a dump over HTTP. -- Inspect-state procedures on other implementations (Cartesi Machine, - bare-metal DEX). -- Cross-implementation determinism test vectors (will land when a - second implementation of the wallet exists to validate against). +The file shares the persistent data directory's +[trust boundary](../threat-model/README.md). Its bytes contain no integrity tag, +checksum, or HMAC. Distribution through a less trusted channel would require +an outer integrity mechanism. diff --git a/docs/watchdog/README.md b/docs/watchdog/README.md index 23f90364..2e34627a 100644 --- a/docs/watchdog/README.md +++ b/docs/watchdog/README.md @@ -1,8 +1,15 @@ # Watchdog -The watchdog is an off-chain safety process that compares the sequencer's -**finalized SSZ state dump** against state produced by the canonical Cartesi -Machine at the same L1 inclusion block. +The watchdog independently replays L1 inputs in the canonical Cartesi Machine +and compares its application-state bytes with the sequencer's accepted +checkpoint at the same L1 block boundary. The wallet's comparison format is SSZ; +the watchdog itself only compares bytes. + +The `/finalized_state` name refers to the sequencer's latest **safe, accepted +batch checkpoint**, not Ethereum's `finalized` tag or a state the watchdog has +already verified. [Snapshot lifecycle](../snapshots/lifecycle.md#acceptance-and-comparison) +owns checkpoint selection and why that application state is comparable at a +whole L1 block boundary. ## Documentation @@ -12,6 +19,7 @@ Machine at the same L1 inclusion block. | **[`getting-started.md`](getting-started.md)** | **Local dev only** — Anvil + `sequencer-devnet`, harness smoke, two-terminal flow | | This file | Architecture, modules, runtime contract, checkpoints, test commands | | [`staging-drills.md`](staging-drills.md) | Webhook smoke, synthetic alarms, staging compare daemon | +| [`design-notes.md`](design-notes.md) | Detection boundaries and checkpoint crash model | | [`sepolia.md`](sepolia.md) | Redirect → [`operator-deployment.md`](operator-deployment.md) | ### Quick start (pick your environment) @@ -45,6 +53,143 @@ overlapping ticks Details: **[`getting-started.md`](getting-started.md)**. +## Runtime Contract + +The watchdog consumes two operator-internal routes: + +- `GET /finalized_state/inclusion_block` — cheap JSON `{ inclusion_block, executed_input_count }` polled every compare tick. +- `GET /finalized_state` — streams the comparison file (`application/octet-stream`); the watchdog reads its `X-Inclusion-Block` and `X-Executed-Input-Count` headers. + +The sequencer's genesis baseline is immediately comparable. A rebuilt baseline +is a restore artifact; these routes return 404 until a new accepted batch +provides a comparison checkpoint. The [snapshot lifecycle](../snapshots/lifecycle.md) +owns availability, response metadata, and artifact leases. + +**Positioning is by L1 block.** If `inclusion_block` equals the watchdog +checkpoint's `safe_block`, the tick exits idle: no state download, L1 fetch, or +CM work. A lower block is a terminal `inclusion_block_regressed` event. For a +higher block, the watchdog replays every InputBox input from `safe_block + 1` +through that block, then downloads the comparison file. If its block header differs from the +polled target, the tick retries rather than comparing different boundaries. + +The client parses `executed_input_count` but does not use it as a replay cursor +or compare it between responses. It does not consume history era/generation +headers or WebSocket events. Its independently replayed L1 checkpoint is +separate from the snapshot-plus-feed protocol described in +[Application history](../protocol/application-history.md). + +The CM's `state` inspect query must return exactly one report. The watchdog +compares those bytes directly with the downloaded file, without decoding or +canonicalizing either side. + +For the toy wallet app, SSZ encoding lives in `examples/app-core/src/wallet_snapshot.rs` +and is shared by `WalletApp::create_dump`, `CanonicalState::canonical_snapshot_bytes`, +and the canonical scheduler's `Inspect` handler (`examples/canonical-app`). + +## Checkpoints + +V1 persists the whole Cartesi Machine checkpoint, including scheduler state; +it does not persist fetched L1 inputs. This is different from the sequencer's +app-owned restore archives (`/latest_snapshot` and `/finalized_snapshot`) and +from its comparison file (`/finalized_state`). The watchdog downloads neither +archive. + +`manifest.json` records `safe_block` (the L1 block through which the CM has +consumed all inputs), timestamp, and optionally the CM image hash. A new +checkpoint directory is written first, then `head.json` is atomically replaced +to point at it. [Design notes](design-notes.md#checkpoint-crash-model) own the +state layout, best-effort pruning, and crash guarantees. + +`init` stores a trusted operator-provided CM snapshot and its declared block +into this layout. That block may precede the sequencer's current comparison +target: `tick` replays the intervening inputs. `init` does not compare against +the sequencer, and a first tick at the same block exits idle; successful init +or idle is not evidence of a state comparison. See the +[accepted detection boundary](design-notes.md#watchdog-state). + +`tick` requires both `config.json` and `head.json`; it never bootstraps from env. +`CARTESI_WATCHDOG_BLOCKCHAIN_HTTP_ENDPOINT` is not persisted in `config.json`, so +operators can rotate RPC endpoints without rewriting watchdog state. It is +required at `tick` for L1 reads, and optionally present at `init` when +auto-detecting `CARTESI_WATCHDOG_BLOCKCHAIN_ID` via `eth_chainId` (prefer setting +the chain id explicitly). + +Bootstrap inputs read by `init`: + +- `CARTESI_WATCHDOG_CM_SNAPSHOT_DIR` +- `CARTESI_WATCHDOG_CM_SNAPSHOT_SAFE_BLOCK` + +## How it runs + +The watchdog has two subcommands: + +```bash +sequencer-watchdog init # setup: writes config.json + head.json (idempotent if complete) +sequencer-watchdog tick # one compare cycle; schedule this +``` + +`tick` does one cycle per process, then exits — infra schedules re-runs +(systemd timer / k8s CronJob) and reacts to the exit code. There is no daemon +loop. `sequencer-watchdog` takes a non-blocking `flock` for `init`/`tick`; +host scheduling should provide the same non-overlap guarantee. A tick follows +the [runtime contract](#runtime-contract), writes a checkpoint only after a +successful comparison, and emits `watchdog_event` on mismatch or regression. +It atomically writes `status.prom` before exit. + +Runtime knobs: + +- `CARTESI_WATCHDOG_BLOCKCHAIN_HTTP_ENDPOINT`: current L1 JSON-RPC endpoint for tick (and optional at `init` for chain-id auto-detect). +- `CARTESI_WATCHDOG_SEQUENCER_URL`: optional tick-time override of the URL persisted at `init` (useful when ephemeral ports change). +- `CARTESI_WATCHDOG_BLOCKCHAIN_ID`: optional chain id label persisted at `init` for `status.prom` (prefer explicit; tick never queries `eth_chainId`). +- `CARTESI_WATCHDOG_METRICS_FILE`: optional override for the Prometheus textfile path (default `$CARTESI_WATCHDOG_STATE_DIR/status.prom`). +- `CARTESI_WATCHDOG_RETRY_ATTEMPTS`: bounded retry attempts per run, default `3`. +- `CARTESI_WATCHDOG_RETRY_DELAY_SEC`: delay between retry attempts, default `5`. + +## Metrics (`status.prom`) + +Each `tick` writes a [Prometheus textfile](https://github.com/prometheus/node_exporter#textfile-collector) +before exiting. Operators scrape or push it from their side — the watchdog does +not run an HTTP server. + +| Exit code | `state` label | Meaning | +|-----------|---------------|---------| +| `0` | `ok` | Compare passed, or idle (finalized unchanged) | +| `1` | `warning` | Retryable failure after retries, or an operator/configuration error | +| `2` | `failed` | State mismatch or inclusion-block regression | + +Gauges (labels `chain`, `app_address` on every series): + +- `cartesi_watchdog_status{state="ok|warning|failed"}` — exactly one series is `1` +- `cartesi_watchdog_divergence_info{kind}` — only on exit `2` + +Exit codes map to `state` only (`0→ok`, `1→warning`, `2→failed`); we do not +export a separate exit-code or last-tick gauge — Prometheus scrape/push already +carries a sample timestamp. + +Set `CARTESI_WATCHDOG_BLOCKCHAIN_ID` at `init` for the `chain` label. If unset, +`init` queries `eth_chainId` from `CARTESI_WATCHDOG_BLOCKCHAIN_HTTP_ENDPOINT` and +persists the result. At `tick`, the env var takes precedence over the persisted value; +the exit path never blocks on RPC (defaults to `unknown` only when neither source +is set). Golden fixtures: [`tests/fixtures/watchdog_status_ok.prom`](../../tests/fixtures/watchdog_status_ok.prom), +[`tests/fixtures/watchdog_status_failed.prom`](../../tests/fixtures/watchdog_status_failed.prom). + +Example after a clean tick: + +```prometheus +cartesi_watchdog_status{app_address="0x4ce...",chain="11155111",state="ok"} 1 +cartesi_watchdog_status{app_address="0x4ce...",chain="11155111",state="warning"} 0 +cartesi_watchdog_status{app_address="0x4ce...",chain="11155111",state="failed"} 0 +``` + +Example Prometheus alert (pull or push gateway — operator choice): + +```promql +cartesi_watchdog_status{state="failed"} == 1 +``` + +Divergence playbook: **notify only**; manual intervention (see +[`operator-deployment.md`](operator-deployment.md)). + ## Host dependencies (`watchdog-lua-deps`) The watchdog cycle and any test that hits HTTP need a native **`lcurl.so`** built into `.deps/lua/`. JSON is pure Lua (no compile step). @@ -112,7 +257,7 @@ Lua modules: - `metrics.lua`: Prometheus textfile (`status.prom`) built and written each tick. - `retry.lua`: bounded retry helper used by the runtime. - `runner.lua`: one compare cycle — cheap `/finalized_state/inclusion_block` - poll, then (when finalized advanced) L1 fetch, CM replay, SSZ compare, + poll, then (when the accepted checkpoint advances) L1 fetch, CM replay, byte comparison, checkpoint write. - `main.lua`: dispatches `init` and `tick`; `tick` exits `0`/`1`/`2` and writes `status.prom`. @@ -135,136 +280,6 @@ this: its scan floor is the operator-supplied checkpoint and it performs no version witness. Do not copy the app-deployment floor into the Lua side without also porting the version witness that makes it sound. -## Runtime Contract - -The sequencer exposes operator-internal snapshot routes (see `sequencer/src/egress/api/snapshot.rs`): - -- `GET /finalized_state/inclusion_block` — cheap JSON `{ inclusion_block, executed_input_count }` polled every compare tick. -- `GET /finalized_state` — streams the finalized SSZ state file (`application/octet-stream`) with `X-Inclusion-Block` and `X-Executed-Input-Count` headers. - -**Idle optimization:** when `inclusion_block` has not advanced past the watchdog -checkpoint's `safe_block`, the tick returns -immediately — no `/finalized_state` download, no L1 `eth_getLogs`, no CM load/advance/inspect. - -The watchdog compares the finalized SSZ bytes with the bytes returned by CM -inspect. It must not canonicalize either side before deciding pass/fail. - -For the toy wallet app, SSZ encoding lives in `examples/app-core/src/wallet_snapshot.rs` -and is shared by `WalletApp::create_dump`, `CanonicalState::canonical_snapshot_bytes`, -and the canonical scheduler's `Inspect` handler (`examples/canonical-app`). - -## Checkpoints - -V1 persists only the resulting Cartesi Machine checkpoint, not the fetched L1 -inputs. - -```text -state_dir/ - config.json - head.json - status.prom # Prometheus textfile from the last tick (see Metrics below) - run.lock # advisory lock handle; file existence is not lock state - checkpoints/ - 00000000000001234567/ - snapshot/ - manifest.json -``` - -`manifest.json` records `safe_block` (the L1 reference block the CM snapshot -covers — the finalized `inclusion_block`), timestamp, -and optionally the CM image hash. A new checkpoint directory is written first, -then `head.json` is atomically replaced to point at it. - -`init` stores the operator-provided bootstrap CM snapshot into this layout. `tick` -requires both `config.json` and `head.json`; it never bootstraps from env. -`CARTESI_WATCHDOG_BLOCKCHAIN_HTTP_ENDPOINT` is not persisted in `config.json`, so -operators can rotate RPC endpoints without rewriting watchdog state. It is -required at `tick` for L1 reads, and optionally present at `init` when -auto-detecting `CARTESI_WATCHDOG_BLOCKCHAIN_ID` via `eth_chainId` (prefer setting -the chain id explicitly). - -- `CARTESI_WATCHDOG_CM_SNAPSHOT_DIR` -- `CARTESI_WATCHDOG_CM_SNAPSHOT_SAFE_BLOCK` - -## How it runs - -The watchdog has two subcommands: - -```bash -sequencer-watchdog init # setup: writes config.json + head.json (idempotent if complete) -sequencer-watchdog tick # one compare cycle; schedule this -``` - -`tick` does one cycle per process, then exits — infra schedules re-runs -(systemd timer / k8s CronJob) and reacts to the exit code. There is no daemon -loop. `sequencer-watchdog` takes a non-blocking `flock` for `init`/`tick`; -host scheduling should provide the same non-overlap guarantee. Each tick: - -1. Loads the watchdog checkpoint from `head.json`. -2. Polls `/finalized_state/inclusion_block`. If it has not advanced past a - watchdog checkpoint, exits `0` (idle). Otherwise: -3. Streams and decodes `InputAdded` logs for the new block range. -4. Replays each successful L1 partition into the in-process Cartesi Machine, - then inspects with query `state`. -5. Byte-compares the SSZ report against `GET /finalized_state`; on match writes a - new checkpoint, on mismatch emits a `watchdog_event` and exits `2`. -6. Atomically writes `$CARTESI_WATCHDOG_STATE_DIR/status.prom` (or - `CARTESI_WATCHDOG_METRICS_FILE`) before exit. - -Runtime knobs: - -- `CARTESI_WATCHDOG_BLOCKCHAIN_HTTP_ENDPOINT`: current L1 JSON-RPC endpoint for tick (and optional at `init` for chain-id auto-detect). -- `CARTESI_WATCHDOG_SEQUENCER_URL`: optional tick-time override of the URL persisted at `init` (useful when ephemeral ports change). -- `CARTESI_WATCHDOG_BLOCKCHAIN_ID`: optional chain id label persisted at `init` for `status.prom` (prefer explicit; tick never queries `eth_chainId`). -- `CARTESI_WATCHDOG_METRICS_FILE`: optional override for the Prometheus textfile path (default `$CARTESI_WATCHDOG_STATE_DIR/status.prom`). -- `CARTESI_WATCHDOG_RETRY_ATTEMPTS`: bounded retry attempts per run, default `3`. -- `CARTESI_WATCHDOG_RETRY_DELAY_SEC`: delay between retry attempts, default `5`. - -## Metrics (`status.prom`) - -Each `tick` writes a [Prometheus textfile](https://github.com/prometheus/node_exporter#textfile-collector) -before exiting. Operators scrape or push it from their side — the watchdog does -not run an HTTP server. - -| Exit code | `state` label | Meaning | -|-----------|---------------|---------| -| `0` | `ok` | Compare passed, or idle (finalized unchanged) | -| `1` | `warning` | Transient failure after retries | -| `2` | `failed` | Deterministic divergence | - -Gauges (labels `chain`, `app_address` on every series): - -- `cartesi_watchdog_status{state="ok|warning|failed"}` — exactly one series is `1` -- `cartesi_watchdog_divergence_info{kind}` — only on exit `2` - -Exit codes map to `state` only (`0→ok`, `1→warning`, `2→failed`); we do not -export a separate exit-code or last-tick gauge — Prometheus scrape/push already -carries a sample timestamp. - -Set `CARTESI_WATCHDOG_BLOCKCHAIN_ID` at `init` for the `chain` label. If unset, -`init` queries `eth_chainId` from `CARTESI_WATCHDOG_BLOCKCHAIN_HTTP_ENDPOINT` and -persists the result. At `tick`, the env var overrides a missing persisted value; -the exit path never blocks on RPC (defaults to `unknown` only when neither source -is set). Golden fixtures: [`tests/fixtures/watchdog_status_ok.prom`](../../tests/fixtures/watchdog_status_ok.prom), -[`tests/fixtures/watchdog_status_failed.prom`](../../tests/fixtures/watchdog_status_failed.prom). - -Example after a clean tick: - -```prometheus -cartesi_watchdog_status{app_address="0x4ce...",chain="11155111",state="ok"} 1 -cartesi_watchdog_status{app_address="0x4ce...",chain="11155111",state="warning"} 0 -cartesi_watchdog_status{app_address="0x4ce...",chain="11155111",state="failed"} 0 -``` - -Example Prometheus alert (pull or push gateway — operator choice): - -```promql -cartesi_watchdog_status{state="failed"} == 1 -``` - -Divergence playbook: **notify only**; manual intervention (see -[`operator-deployment.md`](operator-deployment.md)). - ## Local Tests | Command | What it exercises | diff --git a/docs/watchdog/design-notes.md b/docs/watchdog/design-notes.md index 91fe6ebe..5c2e9ec8 100644 --- a/docs/watchdog/design-notes.md +++ b/docs/watchdog/design-notes.md @@ -1,7 +1,7 @@ # Watchdog Design Notes The watchdog is an independent off-chain safety monitor. It advances a -canonical Cartesi Machine from L1 inputs, inspects the resulting SSZ snapshot, +canonical Cartesi Machine from L1 inputs, inspects its application-state bytes, and byte-compares it with the sequencer's `GET /finalized_state` response at the same finalized `inclusion_block`. @@ -30,7 +30,7 @@ Each tick: 3. Exits cheaply if the finalized block is unchanged. 4. Fetches L1 `InputAdded` logs for the open block range. 5. Advances the CM, inspects state, fetches `GET /finalized_state`, and compares - raw SSZ bytes. + raw bytes (SSZ for the wallet). 6. Writes a new checkpoint only after a successful compare. There is no advance-only mode. Advancing the CM is just an implementation step @@ -48,7 +48,7 @@ safe-input sync. For every at/above-anchor landing the mirrored scheduler accepts, it requires a byte-identical valid local sealed batch at that nonce. A foreign or mismatched landing persists `canonical_divergence`, which freezes the accepted frontier -and finalized-snapshot promotion +and the selection of newer accepted comparison checkpoints ([I15](../invariants.md#i15-divergence-marker-present--acceptance-frontier-frozen)). The offending landing therefore normally never produces a newer `/finalized_state/inclusion_block` for the watchdog to compare. Under the diff --git a/docs/watchdog/getting-started.md b/docs/watchdog/getting-started.md index 17d85058..ab74e9eb 100644 --- a/docs/watchdog/getting-started.md +++ b/docs/watchdog/getting-started.md @@ -27,7 +27,7 @@ Step-by-step guide for running the watchdog alongside a **local** `sequencer-dev | Process | Role | |---------|------| | **Anvil** | Local L1 with Cartesi rollups contracts pre-deployed (`just setup`) | -| **sequencer-devnet** | Off-chain sequencer (wallet app, batches, snapshot promotion) | +| **sequencer-devnet** | Off-chain sequencer (wallet app, batches, accepted comparison checkpoints) | | **watchdog** | Polls `/finalized_state/inclusion_block`, replays L1 inputs in CM, compares SSZ to `/finalized_state` | The sequencer exposes (operator-internal, same HTTP listener today): @@ -109,9 +109,13 @@ A brand-new Anvil history still needs a **fresh** `$CARTESI_WATCHDOG_STATE_DIR` (e.g. `rm -rf /tmp/watchdog-state-devnet`) and a new `init` — old checkpoints won't match. -### Wait for finalized snapshot +### Check comparison availability -The watchdog needs a **finalized** SSZ dump. Right after boot, the cheap endpoint may return **404** until the sequencer has promoted a snapshot. +The watchdog needs a comparable checkpoint from `/finalized_state`. A fresh +genesis setup already provides one at block zero. A rebuilt baseline is not a +comparison checkpoint: after cockroach recovery these routes return **404** +until a new batch is accepted. See +[snapshot selection](../snapshots/lifecycle.md#acceptance-and-comparison). In another shell (use the printed `CARTESI_WATCHDOG_SEQUENCER_URL`): @@ -119,7 +123,11 @@ In another shell (use the printed `CARTESI_WATCHDOG_SEQUENCER_URL`): curl -s "$CARTESI_WATCHDOG_SEQUENCER_URL/finalized_state/inclusion_block" ``` -When you see JSON like `{"inclusion_block":0,"executed_input_count":0}` (numbers may differ), the watchdog can compare. If it stays 404 for a long time, check sequencer logs in `tests/e2e/results/` and that L1 is mining (devnet Anvil auto-mines by default). +JSON such as `{"inclusion_block":0,"executed_input_count":0}` confirms that the +endpoint has a comparison checkpoint. A tick compares only after the reported +block advances beyond its own CM checkpoint; an equal block exits idle. +Unexpected 404 on a fresh devnet warrants checking the URL and sequencer logs +in `tests/e2e/results/`. Optional — inspect SSZ size: @@ -141,11 +149,17 @@ export CARTESI_WATCHDOG_LUA_DEPS=.deps/lua ./watchdog/sequencer-watchdog tick ``` -Success: exit **0**. If finalized has advanced, stderr ends in `compare pass complete`; if it has not, the tick exits idle after the cheap poll. +Success: exit **0**. If the comparison block has advanced, stderr ends in +`compare pass complete`; if it is unchanged, the tick exits idle after the +cheap poll. `init` and an idle tick do not verify the bootstrap state. -Exit codes from `sequencer-watchdog tick`: **0** clean (or idle — finalized unchanged), **1** transient failure (RPC/CM/network after retries), **2** deterministic divergence (`watchdog_event` emitted on stderr before exit). Each tick writes `$CARTESI_WATCHDOG_STATE_DIR/status.prom` — see [`README.md` — Metrics](README.md#metrics-statusprom). +Exit codes from `sequencer-watchdog tick`: **0** comparison passed or idle, +**1** retries exhausted or operator/configuration error, **2** state mismatch +or inclusion-block regression (`watchdog_event` emitted on stderr). Each tick +writes `$CARTESI_WATCHDOG_STATE_DIR/status.prom` — see +[`README.md` — Metrics](README.md#metrics-statusprom). -The watchdog tick runs **one cycle per process and exits** — re-run it on a timer/cron for continuous monitoring. When `inclusion_block` has not advanced since the watchdog checkpoint, the cycle **skips** L1/CM work (idle-cheap) and exits 0. +The watchdog tick runs **one cycle per process and exits** — re-run it on a timer/cron for continuous monitoring. `sequencer-watchdog` takes a non-blocking `flock`; production schedulers should also prevent overlapping ticks with systemd or Kubernetes CronJob `concurrencyPolicy: Forbid`. @@ -161,7 +175,7 @@ Local paths A–B do **not** apply to public L1. There is no `just devnet-for-wa | You spawn Anvil + `sequencer-devnet` | Sequencer already run by ops | | `canonical-machine-image` (devnet guest) | `canonical-machine-image-sepolia` (today); mainnet guest when released | | Snapshot HTTP on localhost | **Internal** operator network only | -| Genesis bootstrap (`safe_block=0`) usual | Bootstrap must match **current** finalized `inclusion_block` | +| Genesis bootstrap (`safe_block=0`) usual | Trusted CM checkpoint at or before the current comparison block; tick replays the gap | **Sepolia is the dress rehearsal for mainnet** — same checklist, alarms, checkpoint volume, and firewall rules; only chain IDs, RPC URLs, and contract addresses change. @@ -200,9 +214,9 @@ See `watchdog/config.lua` for the full list. | `cartesi Lua module is required` | Install Cartesi Machine; use nix/direnv shell; ensure `cartesi-machine` on `PATH` | | `inspect endpoint not implemented` | Rebuild CM image: `just canonical-build-machine-image` | | CM inspect ~27 bytes / JSON in error | Stale image (old JSON inspect); rebuild: `just canonical-build-machine-image` | -| HTTP 404 on `/finalized_state/inclusion_block` | Sequencer not promoted yet; wait or drive L1 + batches | +| HTTP 404 on `/finalized_state/inclusion_block` | Wrong URL, or no comparable checkpoint after a rebuild; a new accepted batch makes the rebuilt history comparable | | `state_mismatch` at genesis | Wrong `CARTESI_WATCHDOG_CM_SNAPSHOT_*` or stale CM image vs sequencer build | -| `inclusion_block_regressed` | Watchdog state ahead of sequencer (reset state dir or fix bootstrap block) | +| `inclusion_block_regressed` | Watchdog checkpoint is ahead of the advertised comparison block; inspect deployment identity, bootstrap block, and sequencer recovery history before reinitializing | | `flock` lock conflict | Another tick is still running or the scheduler allows overlap. With the container `flock`, a leftover `run.lock` path alone is harmless. | | `could not determine which binary to run` | Use `just test-watchdog-compare-harness` (not bare `cargo run -p rollups-e2e`) | | Harness `87 vs 76` or `27 vs 76` byte mismatch | Stale CM image and/or wrong fixture; see [harness troubleshooting](README.md#troubleshooting-just-test-watchdog-compare-harness) | diff --git a/docs/watchdog/operator-deployment.md b/docs/watchdog/operator-deployment.md index 1202813d..f9b31d86 100644 --- a/docs/watchdog/operator-deployment.md +++ b/docs/watchdog/operator-deployment.md @@ -25,7 +25,10 @@ For **local development only** (Anvil + `sequencer-devnet`, CI smoke tests), use └─────────────┘ └────────────────┘ ``` -The watchdog never substitutes for the sequencer. It reads **finalized SSZ** the sequencer already committed and independently replays L1 through the canonical CM. +The watchdog independently replays L1 through the canonical CM and compares +application-state bytes with the sequencer's accepted checkpoint. The wallet +uses SSZ. The [runtime contract](README.md#runtime-contract) explains block +positioning and how the `/finalized_state` name relates to safe acceptance. --- @@ -43,9 +46,13 @@ Verify snapshot API before CM bootstrap: ```bash curl -sS -o /dev/null -w "%{http_code}\n" "$CARTESI_WATCHDOG_SEQUENCER_URL/finalized_state/inclusion_block" -# expect 200 when a finalized snapshot exists (404 = not promoted yet or wrong host) +# expect 200 when a comparable checkpoint exists ``` +A genesis baseline is comparable immediately. A rebuilt baseline returns 404 +until a new batch is accepted; also check for a wrong host or tier. See +[snapshot selection](../snapshots/lifecycle.md#acceptance-and-comparison). + ### 2. Watchdog runtime (release image or local build) **Production (recommended):** pull the **release container image** for tag `vX` — same @@ -156,7 +163,7 @@ Today `WalletApp::default()` / `WalletConfig::sepolia()` align with Sepolia stag | `CARTESI_WATCHDOG_CONTRACTS_INPUT_BOX_ADDRESS` | InputBox on that L1 ([Cartesi deployed contracts](https://docs.cartesi.io/cartesi-rollups/2.0/deployment/self-hosted.md); same lowercase normalization) | | `CARTESI_WATCHDOG_STATE_DIR` | Persistent volume on watchdog host. If the path embeds an address, use **lowercase** — Linux paths are case-sensitive and EIP-55 vs lowercase create sibling dirs | | `CARTESI_WATCHDOG_CM_SNAPSHOT_DIR` | Bootstrap CM snapshot (`init` only) | -| `CARTESI_WATCHDOG_CM_SNAPSHOT_SAFE_BLOCK` | L1 block that bootstrap snapshot represents (= finalized `inclusion_block` at bootstrap) | +| `CARTESI_WATCHDOG_CM_SNAPSHOT_SAFE_BLOCK` | L1 block through which the trusted bootstrap CM has consumed all inputs; at or before the current comparison target | | `CARTESI_WATCHDOG_BLOCKCHAIN_ID` | Chain id label for `status.prom` metrics (prefer set at `init`; optional auto-detect via `eth_chainId` when L1 endpoint is present at `init`) | | `CARTESI_WATCHDOG_METRICS_FILE` | Override path for the Prometheus textfile written by each `tick` | | `CARTESI_WATCHDOG_LUA_DEPS` | `.deps/lua` | @@ -166,21 +173,31 @@ The sequencer discovers and pins `input_box_address` at startup; use the same va ### 5. Initialize watchdog state (first run on a live chain) -On a long-lived deployment, **`CARTESI_WATCHDOG_CM_SNAPSHOT_SAFE_BLOCK=0` is usually wrong** unless finalized state is still at genesis. +The bootstrap must be a trusted **whole CM checkpoint** after all inputs +through its declared L1 block. It need not match the sequencer's current +comparison block: tick replays the gap. `SAFE_BLOCK=0` is valid for a genesis +image even on an old deployment, but replaying the full history can be costly. Pick one: -1. **Ops hands off** a CM snapshot directory + block number matching current finalized `inclusion_block`, or +1. **Ops hands off** a trusted CM snapshot directory + its covered block number, or 2. **Watchdog reuses** `CARTESI_WATCHDOG_STATE_DIR` from a prior run on this deployment, or -3. **Replay from genesis** (only for new rollups / low block height — slow). +3. **Replay from genesis** (potentially slow). + +The sequencer's `/finalized_state` comparison file and app-owned snapshot +archives are different artifacts; neither is automatically a CM bootstrap. +`init` trusts the supplied checkpoint and block without comparing them against +the sequencer. If the first tick sees that same block, it exits idle. See +[the detection boundary](design-notes.md#watchdog-state). Run `init` once to store the bootstrap CM snapshot into the watchdog state layout. Re-running `init` on a **complete** already-initialized state directory is a no-op success (exit `0`), matching `sequencer setup` — safe for process -supervisors that always invoke init before tick. If `head.json` exists but -`config.json` or the selected snapshot is missing/corrupt, `init` fails (exit -`1`) and asks you to wipe `state_dir` and re-run — it will not certify an -unusable state. The L1 RPC URL is not persisted — each `tick` reads +supervisors that always invoke init before tick. It validates the saved +metadata and that the selected snapshot directory is nonempty; it does not +reload an existing CM checkpoint to prove it usable. Missing or malformed +metadata and missing/empty snapshots fail with exit `1`. The L1 RPC URL is not +persisted — each `tick` reads `CARTESI_WATCHDOG_BLOCKCHAIN_HTTP_ENDPOINT` so it can rotate without editing state. If `CARTESI_WATCHDOG_BLOCKCHAIN_ID` is unset at `init`, auto-detect also needs that endpoint present then (prefer setting the chain id explicitly): @@ -202,8 +219,8 @@ budget. `checkpoints//`. 2. If `head.json` is missing: run `sequencer-watchdog init` with `CARTESI_WATCHDOG_CM_SNAPSHOT_DIR` and `CARTESI_WATCHDOG_CM_SNAPSHOT_SAFE_BLOCK` - set to a CM snapshot whose safe block equals the sequencer's current - finalized `inclusion_block`, then run `tick`. + set to a trusted CM checkpoint and its covered block, at or before the + current comparison target, then run `tick`. 3. Schedule `sequencer-watchdog init && sequencer-watchdog tick` (init is a no-op when state is already complete). 4. If `head.json` exists but `config.json` or the selected snapshot is @@ -224,7 +241,7 @@ last-tick gauges — Prom already timestamps samples). Set `CARTESI_WATCHDOG_BLOCKCHAIN_ID` at `init` so `chain` is labeled. If unset, `init` queries `eth_chainId` from `CARTESI_WATCHDOG_BLOCKCHAIN_HTTP_ENDPOINT` and -persists the result. At `tick`, the env var overrides a missing persisted value; +persists the result. At `tick`, the env var takes precedence over the persisted value; the exit path never blocks on RPC (falls back to `unknown` only when neither source is set). @@ -262,7 +279,7 @@ Prometheus push/pull) or on the process exit code. If the process is killed mid-tick, `status.prom` keeps the last completed value until the next run. ```bash -sequencer-watchdog tick # exit 0 = clean/idle, 1 = transient, 2 = divergence +sequencer-watchdog tick # 0 = passed/idle, 1 = retry or operator error, 2 = mismatch/regression ``` `sequencer-watchdog` wraps `init` and `tick` with a non-blocking `flock` on @@ -271,7 +288,8 @@ dies. Use the scheduler's non-overlap primitive as well (for example systemd or Kubernetes CronJob `concurrencyPolicy: Forbid`). A leftover `run.lock` path is only a lock handle; by itself it does not mean a lock is held. -When `inclusion_block` ≤ the watchdog checkpoint, the runner only hits `/finalized_state/inclusion_block` and skips L1/CM work. +An unchanged `inclusion_block` exits idle; a lower block reports +`inclusion_block_regressed` and exits `2`. Both skip L1/CM work. --- @@ -299,7 +317,7 @@ export CARTESI_WATCHDOG_APP_ADDRESS="0x..." export CARTESI_WATCHDOG_CONTRACTS_INPUT_BOX_ADDRESS="0x..." export CARTESI_WATCHDOG_STATE_DIR="/var/lib/watchdog/state-sepolia" export CARTESI_WATCHDOG_CM_SNAPSHOT_DIR="/path/to/canonical-machine-image-sepolia" -export CARTESI_WATCHDOG_CM_SNAPSHOT_SAFE_BLOCK="" +export CARTESI_WATCHDOG_CM_SNAPSHOT_SAFE_BLOCK="" export CARTESI_WATCHDOG_LUA_DEPS="/path/to/sequencer/.deps/lua" ``` @@ -308,7 +326,7 @@ export CARTESI_WATCHDOG_LUA_DEPS="/path/to/sequencer/.deps/lua" If your team runs the sequencer on Sepolia (not only the public endpoint): 1. `sequencer` / release binary with Sepolia `CARTESI_SEQUENCER_*` (chain id, app address, batch submitter key, L1 RPC). -2. Inclusion lane promotes finalized snapshots when L1 safe advances — required for `/finalized_state` 200. +2. Safe accepted batches select comparison checkpoints; genesis is also comparable. A rebuilt baseline needs a new accepted batch before `/finalized_state` returns 200. 3. Snapshot routes on an **internal** bind / port reachable by the watchdog host. 4. Sequencer binary built with **`WalletApp::new(WalletConfig::sepolia())`** (see `sequencer-devnet` vs production binary choice in your release pipeline). @@ -323,7 +341,7 @@ When the rollup runs on Ethereum mainnet, **reuse the same operator checklist ab | L1 RPC | Production-grade archive provider; rate limits matter for wide `getLogs` ranges | | Contracts | Mainnet InputBox, application, portals from production deployment manifest | | CM image | Build from production app/scheduler artifacts (mainnet wallet constants when defined in app-core) | -| Schedule cadence | A cron/timer interval of 300s+ is fine; finalized promotion follows mainnet safe head | +| Schedule cadence | Choose the alert delay you can tolerate; new comparison checkpoints follow safe batch acceptance | | Security | Stricter firewall between public ingress and internal snapshot tier; secrets management for RPC credentials | | Bootstrap | Almost always ops-provided CM snapshot or continued state dir — not genesis replay | @@ -333,35 +351,35 @@ There is no `just devnet-for-watchdog` or automated harness on mainnet; treat Se ## Compare Cycle Behavior (All Live Chains) -Same on Sepolia and mainnet: - -1. Load watchdog checkpoint from `head.json`. -2. `GET /finalized_state/inclusion_block` — if unchanged, **stop** (cheap). -3. If advanced: `eth_getLogs` on InputBox for `(last_block+1)..inclusion_block`. -4. Advance CM incrementally; `inspect` → SSZ bytes. -5. `GET /finalized_state` → SSZ bytes. -6. Raw compare; emit `watchdog_event` + non-zero exit on mismatch. -7. Write new CM checkpoint on success. - -Details: [`README.md`](README.md), [`docs/snapshots/lifecycle.md`](../snapshots/lifecycle.md). +The [runtime contract](README.md#runtime-contract) is the same on Sepolia and +mainnet: poll the comparison block, replay new inputs, compare bytes at the +same block, then save a checkpoint. It defines idle, regression, and retry +behavior when the comparison target moves during a tick. --- ## Checkpoint disk usage and backups -Each successful promotion stores a full CM snapshot under -`$CARTESI_WATCHDOG_STATE_DIR/checkpoints//`, and the watchdog **keeps only -the selected one** — after the atomic `head.json` flip it deletes the -checkpoint it superseded (crash-safe: `head.json` always names a complete -checkpoint). Local disk therefore stays bounded at a single snapshot; no -operator cleanup is required. - -For backups / rollback history, schedule the watchdog tick (it runs one cycle and -exits) and **after it exits** `aws s3 sync $CARTESI_WATCHDOG_STATE_DIR/checkpoints/ -s3://…` (without `--delete`). Because the process has exited there is no race -with its store or prune, and omitting `--delete` **accumulates a per-block -history in S3** while local disk stays at one snapshot. Restore feeds a chosen -snapshot back through the watchdog/sequencer recovery workflow. +Each successful comparison stores a full CM snapshot under +`$CARTESI_WATCHDOG_STATE_DIR/checkpoints//`, then atomically replaces +`head.json` and attempts to delete its predecessor. Pruning is best effort: +failed or interrupted writes/prunes can leave extra directories, so disk use +is not strictly bounded to one snapshot. The +[checkpoint crash model](design-notes.md#checkpoint-crash-model) owns the +pointer-swap sequence and its current lack of file/directory fsync. + +Run tick and backup sequentially in the **same nonoverlapping scheduled job**, +so the next tick cannot store or prune while backup is reading. For a complete +watchdog-state backup, retain `config.json`, `head.json`, and the selected +checkpoint together. Alternatively, syncing `checkpoints/` to object storage +without deletion accumulates a per-block CM history; restore a chosen snapshot +with its manifest's block through watchdog `init`. + +Sequencer recovery takes a native application archive with an acceptance +receipt, exported by `/finalized_snapshot`. Do not treat a watchdog CM +checkpoint as that archive. The +[snapshot backup workflow](../snapshots/lifecycle.md#http-and-recovery-exports) +owns this separate restore path. ## Sequencer restart policy @@ -410,8 +428,8 @@ unclassified restart-with-backoff. Operational notes: |---------|----------------| | `/finalized_state` missing on public URL | Wrong tier — use internal `CARTESI_WATCHDOG_SEQUENCER_URL` | | `failed to load watchdog head` / missing `head.json` | Uninitialized or wiped `STATE_DIR` — see [Missing or corrupt head.json](#missing-or-corrupt-headjson-tick-exit-1) | -| `state_mismatch` | CM image / wallet constants ≠ sequencer build; or wrong bootstrap block | -| `inclusion_block_regressed` | Stale watchdog state vs sequencer finalized head | +| `state_mismatch` | Sequencer/canonical-state disagreement; also verify CM image, wallet constants, and trusted bootstrap block | +| `inclusion_block_regressed` | Watchdog checkpoint is ahead of the advertised comparison block; inspect deployment identity, bootstrap block, and sequencer recovery history before reinitializing | | Slow or failing `getLogs` | RPC range limits — watchdog uses same partition strategy as sequencer | | Transient `L1 RPC latest head lags target block` | Fallback RPC is behind the sequencer's finalized inclusion block; watchdog retries until the node has indexed through the target (avoids truncated `eth_getLogs` false mismatches) | | `inspect endpoint not implemented` | Rebuild CM image for the correct chain target | From 35697691d7a5ba5a2c868f51b3a45c3dd5b6ee44 Mon Sep 17 00:00:00 2001 From: gcdepaula Date: Thu, 17 Sep 2026 15:52:14 -0300 Subject: [PATCH 15/29] docs: retire stale reviews and define review lifecycle --- AGENTS.md | 15 +- README.md | 3 + docs/invariants.md | 4 +- docs/plans/2026-07-coordination-tracks.md | 31 +- docs/plans/2026-07-track6-dump-api-design.md | 218 ----- docs/plans/2026-08-authority-boundary-adr.md | 69 +- docs/protocol/c-application-binding.md | 10 +- .../2026-08-18-over-engineering-review.md | 46 - .../2026-08-22-lifecycle-simplification.md | 60 -- docs/review/2026-09-03-branch-stocktake.md | 847 ------------------ .../2026-09-09-application-lane-dex-review.md | 389 -------- docs/review/2026-09-16-track3-validation.md | 33 +- docs/review/README.md | 60 ++ docs/review/register.md | 832 +++-------------- sequencer-core/src/fee.rs | 46 +- 15 files changed, 295 insertions(+), 2368 deletions(-) delete mode 100644 docs/plans/2026-07-track6-dump-api-design.md delete mode 100644 docs/review/2026-08-18-over-engineering-review.md delete mode 100644 docs/review/2026-08-22-lifecycle-simplification.md delete mode 100644 docs/review/2026-09-03-branch-stocktake.md delete mode 100644 docs/review/2026-09-09-application-lane-dex-review.md create mode 100644 docs/review/README.md diff --git a/AGENTS.md b/AGENTS.md index a64aae4b..92b5cb78 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -341,12 +341,11 @@ Keep three kinds of material distinct: must distinguish proposed behavior from implemented contracts. On completion, put the durable design in its owner and reduce the plan to its remaining work and links. -- **Historical evidence** lives in review ledgers, explicitly marked historical - documents, and commit history. A review ledger is append-only while open. - When it closes, promote conclusions into current docs, record settled and - refuted proposals in the [review register](docs/review/register.md), and remove - process narration. Retained superseded proposals must say they are historical - and link to the current contract; they are evidence, not instructions. +- **Review notes** are temporary working memory. Commit them when they help an + active review or handoff, then distill and delete them when that work ends. + The [review lifecycle](docs/review/README.md) owns the policy: unresolved work + stays in one register or active plan, durable reasoning in its current owner, + completed history in Git. Keep dated evidence only for a named ongoing use. **Record deliberate absence once**, at the seam where someone would re-add the mechanism, phrased as a positive design statement with its reason. Avoid removal @@ -400,7 +399,7 @@ See [Running](README.md#running) for the two-phase `setup` / `run` workflow. - Run at least `cargo check` before finishing. - Read the relevant recovery guide and both current TLA+ models before touching recovery code, and the threat model before touching trust-boundary code. -- Check [`docs/invariants.md`](docs/invariants.md) before changing anything it lists as load-bearing, and [`docs/review/register.md`](docs/review/register.md) for open findings in the code you're about to touch and for decisions already settled or refuted. +- Check [`docs/invariants.md`](docs/invariants.md) before changing anything it lists as load-bearing, the owning design for its assumptions, and [`docs/review/register.md`](docs/review/register.md) for unresolved work in the code you're about to touch. Verify review claims against current code. ### Ask First @@ -451,4 +450,4 @@ work reaches another boundary. | Submission fees or oracle pricing | [L1 fee policy](docs/l1-fee-policy.md) — estimation and replacement limits; [threat-model actor table](docs/threat-model/README.md#actors-and-trust) — oracle source and outage assumptions. | | Command setup or deployment configuration | [Running](README.md#running) and [config.rs](sequencer/src/commands/config.rs) — invocation, identity pinning, defaults, and validation. | | Watchdog development or operation | [Architecture](docs/watchdog/README.md); [local dev](docs/watchdog/getting-started.md) for Anvil; [operator deployment](docs/watchdog/operator-deployment.md) for Sepolia/mainnet. | -| A new mechanism, simplification, or work spanning an active track | [Review register](docs/review/register.md) — relevant open findings and settled/refuted reasoning; [coordination tracks](docs/plans/2026-07-coordination-tracks.md) — remaining work and dependencies. Consult dated evidence when its reasoning is needed. | +| A new mechanism, simplification, or work spanning an active track | Owning design and [invariants](docs/invariants.md) — reasons and assumptions; [review register](docs/review/register.md) — unresolved work; [coordination tracks](docs/plans/2026-07-coordination-tracks.md) — priorities and dependencies. Follow the [review lifecycle](docs/review/README.md) when recording conclusions. | diff --git a/README.md b/README.md index 8dfc7cfe..72a83068 100644 --- a/README.md +++ b/README.md @@ -240,6 +240,9 @@ After each successfully applied input at offset `X`, persist the claim with including business failures and malformed-direct no-ops. - Recovery stops the process and disconnects subscribers. A reconnect must present its saved claim; offsets alone cannot distinguish a replaced suffix. +- Shutdown or a feed read/send failure may disconnect without a WebSocket + Close frame. Resume from the saved claim after an unexpected disconnect; + a clean close is not required for safe replay. Message shapes: diff --git a/docs/invariants.md b/docs/invariants.md index f9e44ac9..422aafff 100644 --- a/docs/invariants.md +++ b/docs/invariants.md @@ -371,7 +371,9 @@ by writer and are write-once (`0001_schema.sql`). same anchor via `open_fresh_tip_in_tx`'s `parent = None` path, after invalidating the old root — so only one *valid* parentless root ever exists, invalidated ones coexisting. -- **Enforced by:** `trg_enforce_nonce_contiguity` — its parentless arm is an +- **Enforced by:** the parent foreign key (enabled on every writer) rejects + dangling parents; `trg_enforce_nonce_contiguity` checks nonce succession. + Its parentless arm is an *exact* match `nonce == (SELECT nonce FROM batch_tree_anchor)` (tighter than a bare "must be 0"), plus an at-most-one-valid-parentless-root guard scoped to `invalidated_at_ms IS NULL`; `compute_next_nonce(None)` reads the diff --git a/docs/plans/2026-07-coordination-tracks.md b/docs/plans/2026-07-coordination-tracks.md index b4781d77..d10ffd43 100644 --- a/docs/plans/2026-07-coordination-tracks.md +++ b/docs/plans/2026-07-coordination-tracks.md @@ -2,7 +2,7 @@ **Status:** active plan of record. Tick / annotate as work lands; when a track completes, move its durable outcomes into the normative docs and -collapse its entry here. +remove its entry here. Track numbers retain their existing identities. Context: Bart is building **libdex**, a native (non-CM) app whose backing storage is an mmap'd flat buffer, and will reimplement the scheduler in C++. @@ -12,14 +12,9 @@ freely at this stage — no backward-compatibility constraints. | # | Track | Owner | Status | |---|-------|-------|--------| -| 1 | WS context fields + L1 provenance (PR #26) | Stephen | **done** — merged to main | -| 2 | Restore `docs/review/` ledger + this plan | us | **done** | | 3 | Feed & replay protocol redesign | us (design) → us/Stephen (impl) | **implemented** — canonical application history, snapshot restore archives, mandatory WS claims, typed refusals, and SDK cutover; [remaining integration gates](2026-07-track3-feed-replay-design.md#remaining-integration-gates) | -| 4 | Storage decode policy | us | **done** — fail-loud for contract-impossible values; the named `saturating_query_bound` only where clamping preserves the predicate (policy lives in `storage/convert.rs` + the invariants check policy) | | 5 | Fee exponentiation LUT | us | **deferred** — decided exact-floor if built (the table *is* the spec, algorithm-free; replay continuity across the upgrade explicitly not preserved); a separate pending design decision may make log-space fees defunct — revisit after syncing with Bart | | 6 | Dump / `Application` API redesign | us + Bart | **interface and reference C binding implemented** — [Application contract](../protocol/application-contract.md); native-engine integration gates remain | -| 7 | LLM context-engineering review | us | **done** — skills/agents/settings homed in-tree; the docs-practice rules live in AGENTS.md | -| 8 | Runtime ownership and terminal stop | us | **done** — owned by the [authority-boundary ADR](2026-08-authority-boundary-adr.md) | **Current campaign order:** @@ -67,8 +62,22 @@ implemented. End-to-end native snapshot-to-live bootstrap remains an integration gate, alongside the private DEX engine when available. Reference bridge conformance cannot establish private-engine correctness. -The [July proposal](2026-07-track6-dump-api-design.md) is historical; the -[September review](../review/2026-09-09-application-lane-dex-review.md) records the -accepted simplifications. Additional public checkpoint primitives or asynchronous -scheduling need a measured requirement. Watchdog extraction from the DEX's -canonical state drive remains separate work. +Remaining checks need the actual consumer: + +- Exercise snapshot-to-live bootstrap and canonical comparison through the C + host in CI; its current smoke test builds and invokes `--help`. A reusable + conformance runner needs engine-supplied genesis and meaningful accepted and + rejected inputs. Compare canonical state files, not recovery-dump layouts. +- Verify the external scheduler's ordering, fee conversion, and recovery + agreement. Publish independent-port fee vectors for the + [current arithmetic](../../sequencer-core/src/fee.rs); a deferred LUT is a + separate semantic change. Watchdog extraction from the DEX's canonical state + drive remains integration work. +- Decide output storage, checkpoint layout, ABI version negotiation, generated + bindings, and linker policy from concrete engine requirements. The current + drain protocol and path callback remain the contract until then. + +Additional checkpoint primitives or asynchronous scheduling need a measured +requirement. Any future microbatch priority scheme must preserve per-account +nonce order, serial execution, and the frame-time contract; the current lane +does not promise priority scheduling. diff --git a/docs/plans/2026-07-track6-dump-api-design.md b/docs/plans/2026-07-track6-dump-api-design.md deleted file mode 100644 index 528ce0a7..00000000 --- a/docs/plans/2026-07-track6-dump-api-design.md +++ /dev/null @@ -1,218 +0,0 @@ -# Historical dump / `Application` proposal (Track 6) - -**Status: superseded by the 2026-09-09 design decision.** Preserved as the -original proposal; its public clone/flush machinery was not adopted. The -[current Application contract](../protocol/application-contract.md) specifies -mutable checkpoint creation, independent restore, and the distinction between -recovery checkpoints and canonical comparison bytes. The -[review ledger](../review/2026-09-09-application-lane-dex-review.md) records the -reasoning. The text below is historical. - -## 1. Motivation - -Three forces, one API: - -- **Cost.** `create_dump(&self)` is a full O(state) serialize + 3-fsync - ladder, run synchronously on the single lane thread at every batch close — - soft-confirmation acks stall for the duration. Tolerable for the toy - wallet; not for a multi-GB flat buffer. -- **Bart's app shape.** libdex runs natively against an mmap'd flat state - buffer — the Cartesi Machine's own storage approach (Bart implemented that - feature), whose ecosystem exploits CoW (`clone_stored`: reflink, plus - hardlinks — safe there only because stored images are immutable, see §10 — - with plain-copy fallback; ~2.6 ms vs ~350 ms full store at 533 MB in - Dave's measurements). -- **Verb review.** `create_dump / from_dump / delete_dump / - state_file_in_dump` is a leaky projection of what the lane needs; the - 2026-07 investigation mapped it against the CM/Dave verb set. - -Key investigation results this design builds on: the CM emulator has **no -commit/revert** — those are node-level orchestration over `clone_stored`, -and the sequencer already has both (DB row as commit point; older-dump + -replay as revert) correctly *off* the trait. The one missing primitive is -**cheap clone**. And the CM has since moved our way on durability: -machine-emulator PR #398 adds `rename_stored` and makes it and -`remove_stored` **durable (auto-synced)** — the commit-point idiom our dump -lifecycle hand-rolls. - -## 2. Requirements - -From the lane trace (R) and the wishlist (W): - -- **R1** Checkpoint the current state at batch close — atomic, - crash-durable *before* the DB row lands (invariant I13). -- **R2** Reconstruct at startup: latest checkpoint + replay. -- **R3** Dispose checkpoints (GC + orphan sweep). -- **R4** Serve canonical bytes over HTTP without instantiating the app; - bytes must equal the canonical machine's `inspect_state` output (the - watchdog byte-compares). -- **R5** Genesis construction (off-trait today, stays off-trait). -- **W1** Checkpoints cheap enough to not shape batch policy (CoW). -- **W2** Checkpointing off the ack path. -- **W3** Natural fit for an mmap'd-working-image app without penalizing - pure-RAM apps (WalletApp). - -## 3. The fork, decided: working-image model - -The investigation flagged one load-bearing fork: cheap clones require the -app to run against an on-disk image the lane can flush-then-clone (Dave's -`SHARING_ALL` model); `create_dump(&self)` serializing live RAM can never be -cheap. **This design takes the working-image model** — it is libdex's -natural shape, it is what the CM ecosystem optimizes, and WalletApp adapts -trivially (its "working image" is a file it rewrites on flush; cost -unchanged from today's serialize). - -## 4. Proposed trait - -```rust -pub trait Application: Send + Sized { - // --- execution surface unchanged --- - - /// Open the app on a working image directory. The sequencer owns the - /// directory's lifecycle; the app owns its contents. Called at startup - /// (from a cloned checkpoint) and after genesis materialization. - fn open(working: &Path) -> Result; - - /// Make the working image consistent and durable on disk: flush - /// app-level caches, msync mapped pages, fsync files. After `Ok`, the - /// on-disk image alone reconstructs this exact logical state via - /// `open`. Called by the lane at batch close, before cloning. - fn flush(&mut self) -> Result<(), AppError>; - - /// Clone the (flushed, not currently open) image at `from` into `to` - /// (must not exist). Default: recursive plain copy + fsync ladder. - /// CoW apps override with reflink/hardlink (FICLONE / clonefile), - /// keeping the same durability contract: on `Ok`, `to` survives an - /// immediate kernel crash. - fn clone_image(from: &Path, to: &Path) -> Result<(), AppError> { … } - - /// Delete an image directory the sequencer no longer references. - /// Default: remove_dir_all. - fn delete_image(prefix: &Path) -> Result<(), AppError> { … } - - /// Locate the single canonical state file inside an image without - /// opening the app. Contract unchanged from state_file_in_dump: - /// the file's bytes equal canonical `inspect_state` output (R4). - fn canonical_file_in_image(prefix: &Path) -> PathBuf; -} -``` - -Verb mapping: `open` ≈ CM `load(SHARING_ALL)`; `flush` ≈ the msync the CM's -dirty-page sidecars make optional; `clone_image` ≈ `cm_clone_stored`; -`delete_image` ≈ `remove_stored`. Commit stays sequencer-side (DB row; -sealing by durable rename per PR #398's precedent). Revert stays -sequencer-side (older checkpoint + replay). `from_dump`/`create_dump` -disappear: restore is `clone_image(checkpoint, working)` + `open(working)`; -checkpoint is `flush()` + `clone_image(working, checkpoint)`. - -## 5. Lane lifecycle changes - -Batch close becomes: `flush()` → `clone_image(working, staging)` → -sequencer writes `info.toml` + durable-renames staging into the dumps dir → -DB row in one tx (unchanged commit point). Filesystem-first ordering, the -"orphan dir possible, dangling row never" invariant, promotion, GC, leases, -and the whole `snapshot_dumps.rs` layer are **unchanged** — the storage half -is already representation-agnostic (opaque prefix keys). - -W2 (off-ack-path) falls out for CoW apps: `flush` + reflink is -milliseconds, and the expensive part (page write-back) is the kernel's -business afterward. A dedicated async stage is *not* designed in; if a -non-CoW app's flush is slow, that is the app's cost to fix by adopting CoW. - -Startup (R2): clone the promoted/pending checkpoint into a fresh working -dir, `open`, replay. The working dir is disposable state — never promoted, -never served, deleted on clean start. - -## 6. Crash-safety posture (unchanged, now sharper) - -I13 stays: nothing may reach the DB before the corresponding image is -durable. The split is now explicit: the **app** guarantees durability of -image *contents* (`flush`, `clone_image`); the **sequencer** guarantees -durability of *directory structure* (rename + dir-fsync — exactly what CM -PR #398 now bakes into `rename_stored`/`remove_stored`, validating the -posture). Tell Bart directly: the CM's historical no-fsync stance does not -apply here — a CoW `clone_image` override must fsync what reflink leaves -unsynced, and #398 shows the CM itself now agrees for the rename/remove -verbs. - -## 7. Serving (R4) and the libdex layout constraint - -`canonical_file_in_image` keeps the single-canonical-file contract — the -HTTP snapshot routes, lease protocol, and watchdog byte-compare all survive -untouched. The constraint to put in front of Bart *before* libdex's layout -freezes: the served file must byte-match canonical `inspect_state` output, -so either (a) the flat buffer's layout is itself canonical — fully -normalized, no allocator padding, no free-lists, no pointer-valued fields, -no uninitialized gaps — and the buffer file doubles as the canonical file; -or (b) libdex writes a separate canonical projection during `flush`, which -reintroduces O(state) serialize cost and partly defeats CoW. (a) is the -performant answer and a real design constraint on his buffer format. - -## 8. Migration & cleanups folded in - -- **WalletApp:** `open` mmap-or-reads its file; `flush` = today's - serialize+fsync ladder; defaults cover the rest. No capability lost. -- **Genesis:** concrete-type constructor materializes the initial working - image, then the normal flush/clone path checkpoints it (R5 unchanged). -- **`SafeInputRecord` shim** (`storage/l1_inputs.rs`): collapse - `StoredSafeInput`/`IngestedSafeInput` into one honest row model in a dedicated - cleanup. The provenance/clock decision below is settled; it no longer blocks - that cleanup. -- **Direct-input clock semantics (settled with Track 3):** `block_timestamp` - is persisted and served as feed provenance only; it is not an application - transition input. Directs execute at their exact L1 inclusion block and user - ops at their frame's safe block, as owned by the - [`Application` contract](../protocol/application-contract.md#3-the-safe-block-clock--last_executed_safe_block). - No timestamp-bearing trait change is needed, and this no longer blocks - libdex's state design. - -## 9. Open questions - -1. **Working-image locking:** the CM uses flock to make `SHARING_ALL` - exclusive. Do we require the app to hold an equivalent lock, or does the - sequencer's single-lane discipline suffice? (Lean: sequencer discipline - suffices; a lock is cheap defense — app's choice.) -2. **`flush` durability scope:** must `flush` fsync, or is fsync deferred to - `clone_image`? (Lean: `clone_image` owns durability of the *clone*; - `flush` owns consistency of the *source* — msync yes, fsync optional.) -3. **Non-reflink filesystems:** plain-copy fallback makes checkpoint cost - O(state) again silently. Log loudly at startup when the dumps dir does - not support reflink? (Lean: yes — one probe at bootstrap.) -4. **Dirty-page tracking for hashing/inspect:** the CM's `.dpt` sidecars - make post-clone hashing touch only dirty pages. libdex would need its - own write-barrier to replicate; out of scope for the sequencer API but - worth flagging to Bart as a cost driver for (a) in §7. - -## 10. Review remarks (non-normative, open design) - -These remarks record issues raised during review; they do not replace the -proposed trait or lifecycle above. (An earlier revision promoted the first -remark into §4's `clone_image` contract and misattributed that promotion to -a maintainer instruction; the 2026-08-01 review corrected the record and the -remarks-only posture is restored. Whether hardlinks stay in §4's suggestion -is settled in design review with Bart, alongside the rest of the trait.) - -- Hardlinks are not a valid implementation of the mutable-working-image to - immutable-checkpoint clone. Later in-place or mmap writes through either - path mutate the same inode and therefore the checkpoint. Any accepted - implementation should require a real CoW clone (`FICLONE`/`clonefile`) or - an independent copy, with a test that mutating the reopened working image - cannot change checkpoint bytes. -- Synchronous durability is likely simple and fast enough, and should remain - the baseline while it is measured. Measure the relevant phases separately: - app flush/msync, source synchronization if required, clone or copy, - destination data and metadata synchronization, staging rename, parent - directory synchronization, and DB commit. Measurements should cover - representative state sizes and dirty-page ratios and report tail latency, - not only a best-case reflink time. -- The contract that `clone_image -> Ok` survives an immediate crash is - stronger than the statement in §5 that expensive page writeback may remain - the kernel's work afterward. Until filesystem-specific ordering and sync - requirements demonstrate both claims together, deferred writeback is an - unresolved durability issue rather than work safely removed from the - checkpoint path. -- An asynchronous checkpoint stage need not be designed preemptively. If the - synchronous phase measurements meet the latency budget, keeping the - durability boundary synchronous is preferable. Async staging should be - reconsidered only if measurements show that the required flush and sync - operations materially violate that budget. diff --git a/docs/plans/2026-08-authority-boundary-adr.md b/docs/plans/2026-08-authority-boundary-adr.md index 305e73f4..b2e8672e 100644 --- a/docs/plans/2026-08-authority-boundary-adr.md +++ b/docs/plans/2026-08-authority-boundary-adr.md @@ -2,10 +2,8 @@ The architecture decision record for how authority — over speculative state, promises, process lifetime, and recovery admission — is owned in the -sequencer. The mechanisms below are landed; the decision history and the -review trail that shaped them live in -[`../review/register.md`](../review/register.md) and the review history it -records. +sequencer. The mechanisms and their reasons below describe the current design; +Git preserves the earlier proposals and review history. ## Context @@ -57,18 +55,23 @@ The lock is released only after every runtime-owned child has actually stopped; a dropped `JoinHandle` detaches rather than stops, so each worker and nested blocking task retains its own lock clone until its closure ends. This prevents two processes on one data directory; it is not distributed -fencing. Cleanup polls every worker concurrently, so one hung drain cannot -hide another worker's terminal exit. Ordinary shutdown has no hard deadline. +fencing. Fresh runtime channels and complete worker shutdown separate runs; +an internal fencing epoch would need a new use case such as overlapping +runtimes or same-process hot replacement. Cleanup polls every worker +concurrently, so one hung drain cannot hide another worker's terminal exit. +Ordinary shutdown has no hard deadline. The reader can cancel a pending RPC read, but awaits any started SQLite append before joining, so the final clean-exit divergence check sees every committed sync. -Runtime construction is prepare → admit → launch: every fallible or awaited -operation happens while zero tasks exist; final admission checks one +Runtime construction is prepare → admit → launch: fallible or awaited +dependency preparation happens before workers launch; final admission checks one consistent fact set; launch spawns every worker in one infallible, non-yielding block, consuming the single-use `RuntimeAdmission` witness. A preparation failure cannot leave a partially launched runtime, and no -refusal or retry can mint the witness. +refusal or retry can mint the witness. This boundary does not promise that +worker initialization succeeds: application restore and catch-up run inside +the launched lane. ### 2. Fact-derived admission and the terminal-fault black box @@ -92,15 +95,23 @@ Every fault whose evidence the boot path reads re-refuses before the first soft confirmation; the residual window is recorded in the threat model. The honesty backstops (rollbackable soft confirmations, the watchdog byte-compare, and the divergence freeze) do not depend on a boot gate. +A durable gate on the previous verdict would require operator acknowledgement +without adding evidence about the current facts. A full integrity sweep would +still miss semantic faults outside its read set. Revisit admission checks for +a specific detectable fault, rather than treating a previous verdict as proof. ### 3. Ordered startup recovery Normal `run` startup inspects local terminal facts, syncs L1, selects a repair from current facts, and checks the result. The flush branch orders flush → sync through the returned safe block → cascade explicitly. There is no -phase driver or progress ledger; the flush witness is a local value. +phase driver or progress ledger; the flush witness is a local value. A restart +must obtain fresh flush/sync evidence; persisting a phase would let it skip +work based on an earlier attempt's observation. Setup/rebuild, maintenance flush, and normal-run recovery retain distinct -typed controllers. The dispatch table, boot-local witnesses, and final +typed controllers because they establish different facts; a universal +controller would represent combinations none of those commands needs. +The dispatch table, boot-local witnesses, and final admission check are owned by [`docs/recovery/README.md`](../recovery/README.md); [`admission.tla`](../recovery/admission.tla) verifies the controller ordering. @@ -145,17 +156,14 @@ write-before-broadcast watermark authorizes an L1 submission; committed version-checked application-input rows authorize the feed output. Effects handed to the network before process termination may still complete remotely. -## Rejected alternatives - -`RunEpoch` (an internal fencing epoch); `EffectGate` / `LiveKernel` (a -universal effect mutex or actor); a generic command controller (one reducer -over setup/rebuild/run/maintenance); a -per-chunk divergence query, provider call, or reader mailbox on the hot -path; a durable recovery-phase ledger; a durable boot gate on terminal -verdicts. Each argument, its evidence, and its revisit trigger live in the -review register's refuted list -([`../review/register.md`](../review/register.md#refuted--do-not-re-propose-without-new-evidence)); -do not re-propose without new evidence. +A global effect mutex or actor would duplicate these owners and put authority +into an additional in-memory coordination layer. A per-chunk divergence read +would see only already-detected accepted-batch divergence, not establish that +every soft confirmation will become canonical; adding a provider call would +also couple acknowledgement latency to L1 availability. The supported reaction +and race bound belong to [I15](../invariants.md#i15-divergence-marker-present--acceptance-frontier-frozen). +Revisit this boundary if a new effect needs authority that its existing owner +cannot establish, or the supported guarantee changes. ## External history @@ -175,12 +183,11 @@ suffix. See the [history contract](../protocol/application-history.md) and ## Performance posture -The product contract is `POST /tx` acknowledgement under 500 ms. Same-host -release sweeps across the cutover found no material regression: ACK p99 at -or below ~50 ms through concurrency 256 with zero rejections, concurrency-1 -HTTP ACK p50 around 13 ms (submit-to-matching-WS-event p50 roughly double — -name which metric "round-trip" means). Same-host numbers are method-specific -regression evidence, never capacity claims: at high concurrency the load -clients contend with the sequencer, so the plateau is machine saturation. A -separate-machine load generator is required for capacity measurement, and -round-trip remeasurement belongs with the public history/API projection. +The product contract is `POST /tx` acknowledgement under 500 ms; the +[benchmark specification](../../tests/benchmarks/BENCHMARK_SPEC.md) defines +the evaluation conditions. The [retained comparison](../review/2026-09-16-track3-validation.md) +records exact revisions, workload, and same-host ACK/WS measurements. They are +regression evidence: client/host contention and excluded startup or backlog +work prevent interpreting them as deployment capacity. Representative latency, +including checkpoint and L1-reconciliation overlap, remains an +[integration gate](2026-07-track3-feed-replay-design.md#remaining-integration-gates). diff --git a/docs/protocol/c-application-binding.md b/docs/protocol/c-application-binding.md index adc06e27..49604648 100644 --- a/docs/protocol/c-application-binding.md +++ b/docs/protocol/c-application-binding.md @@ -38,10 +38,12 @@ output and diagnostic buffers; the adapter copies them before reuse. Fee fields carry base-129/128 exponents. The shared max-fee comparison operates in log space; an application checking balances or charging fees uses a linear -amount. The reference conversion lives in -[`sequencer-core/src/fee.rs`](../../sequencer-core/src/fee.rs). Native and -canonical execution must agree on that conversion, since different amounts can -change rejection decisions and resulting balances. +amount. The exact conversion contract lives on `fee_to_linear` in +[`sequencer-core/src/fee.rs`](../../sequencer-core/src/fee.rs): ports must match +the table, intermediate flooring, and ascending bit order, not just the nominal +exponential formula. [`build.rs`](../../sequencer-core/build.rs) generates the +table and exponent bound. Native and canonical execution must agree on the +conversion, since different amounts can change rejection decisions and balances. The same implementation may be compiled for native execution and the canonical machine. This does not establish equivalent behavior across targets: the diff --git a/docs/review/2026-08-18-over-engineering-review.md b/docs/review/2026-08-18-over-engineering-review.md deleted file mode 100644 index 8cb61954..00000000 --- a/docs/review/2026-08-18-over-engineering-review.md +++ /dev/null @@ -1,46 +0,0 @@ -# Over-engineering review (2026-08-18) - -Full-branch review of the authority-boundary + durable-history-foundation -branch against the project's design goals (readable, auditable, every -mechanism judged against its weight): seven parallel subsystem reviews, each -proposal adversarially cross-examined against the invariants register, the -TLA+ models, and git history, plus an independent premise challenge of the -ADR. 141 mechanisms inventoried: 98 keep, 25 simplify, 6 cut, 12 question. -This review adopted the calibration rule now in AGENTS.md (the complexity -budget belongs to concurrency, mutual exclusion, durability, and hostile-L1 -robustness). - -**Verdict: not over-engineered — unevenly engineered.** The ADR's premise -survived attack; the rejected alternatives left no residue in code. Three -findings cut against "smallest sufficient design": the in-process containment -predicate was still convention at ~11 hand-placed sites (fixed by the -`Authorized` token), ~700 lines of mechanical repetition had accumulated -(harvested), and the lifecycle module implemented the right guarantee with a -heavier representation than needed (re-platformed, then removed — see below). - -**Outcomes** (all landed 2026-08-18/19, each wave adversarially re-reviewed -post-commit; per-item dispositions in [`register.md`](register.md)): - -- Eleven defects fixed (fail-open classification arms, admission bypasses, - containment publication window, snapshot-route gating, typed app-boundary - refusals). -- ~700-line harvest: worker-exit plumbing collapsed with terminality beside - each error type; a real `RuntimeScope` with `ShutdownSignal` reduced to its - name; fee-oracle bootstrap behind its module; recovery type-stack trimmed - to the one `RecoveryProgress` enum; dead surface deleted. -- The `Authorized` externalization token: the containment consult became a - compile-time obligation of the effect functions. -- Module homing: command *brackets* to `commands/` (with their config/error - taxonomy, `RunError` → `CommandError`), the capability substrate alone in - `runtime/`, `L1Config` to `l1/`, the clock to the crate root. -- **Lifecycle arc:** re-platform to singleton + audit trail (decision L1) → - admission gating removed entirely, facts govern (decision L2, 2026-08-19) - → journal narrowed to the terminal-fault black box (decision L3, see - [`2026-08-22-lifecycle-simplification.md`](2026-08-22-lifecycle-simplification.md)). - The surviving design rationale lives in the ADR and the invariants check - policy. - -Still open from this review: the 500 ms latency contract does no design work -(nothing is shaped by it — decide what it is *for*), and the -catch-up ACK-latency measurement owed to the benchmark harness. Tracked in -the register. diff --git a/docs/review/2026-08-22-lifecycle-simplification.md b/docs/review/2026-08-22-lifecycle-simplification.md deleted file mode 100644 index 0a925d24..00000000 --- a/docs/review/2026-08-22-lifecycle-simplification.md +++ /dev/null @@ -1,60 +0,0 @@ -# Lifecycle simplification review — L3 (2026-08-22) - -Fresh-eyes review of what remained of the lifecycle machinery after decision -L2 (admission gating removed, 2026-08-19). Method: two exhaustive read-only -sweeps (a 28-class terminal-fault re-detection map; a journal -weight-and-consumers audit), a six-refuter adversarial verification of every -load-bearing claim before acting, and a fifteen-agent post-landing review of -the diff. - -**Findings that grounded the decision:** - -- The L2 characterization was true: admission was exactly three facts, and - the attempt journal had zero production reads for decisions — its ~490 - production lines across ten files bought only what tracing already - provided, plus the terminal-cause row. In `admission.tla` the journal - variable was a bijective ghost of the controller (removing it left TLC - state counts byte-identical). -- The "terminal faults refuse at re-detection, not at boot" trade is far - narrower than it sounds: everything with durable or deterministically - re-derivable evidence re-refuses before the first soft confirmation, and - the batch/frame spine is re-read within seconds of launch. The verified - residual (cold payload bytes below the lane checkpoint, reachable only via - the WS catch-up window or a pending batch's re-encode; faults with no - durable evidence at all) is recorded as an accepted boundary in the threat - model. - -**Decision L3 (landed):** the journal narrowed to the `terminal_faults` -black box — append-only command+cause rows, best-effort. Principle adopted, -now in the invariants check policy: **telemetry writes are verdict-neutral** -— they sat on the brackets' `?` paths and could change exit codes. -`admit_runtime` collapsed to one consistent inspect + reduce; `RunId` and -the event vocabulary went with the settle plumbing. No boot machinery was -added in the journal's place: neither a durable verdict gate (it needs an -acknowledgement to exit, which carries no information the reducer doesn't -re-derive) nor a boot-time full-integrity sweep (expensive, and blind to -semantic violations outside its read set). - -**Verdict-integrity defects fixed with it** (each adversarially confirmed -first): settle-masking (a settle-step failure could replace a terminal -verdict with exit 1 — and settle was the *sole* exit-code determinant for -most terminal paths); `CommandError::Lifecycle` classified wholesale -terminal (a transient `SQLITE_BUSY` on a lifecycle write paged); signer -misconfiguration classified as unclassified I/O in all three keyed commands. -The misconfig-poison taxonomy question from the 2026-08-18 review closed -with L3: there is no poison to apply; what remained was exit-code accuracy. - -**Post-landing review (8 confirmed / 4 refuted):** one real regression — the -Ok-path divergence refusal had been dropped with `settle_clean`, letting a -clean drain over freshly persisted divergence exit 0 (the one code that -breaks the supervisor's restart-then-refuse rediscovery chain); restored as -an explicit fact check and test-pinned. One missing test pin added; six -doc-staleness items fixed. A claimed re-detection gap at -`finalized_snapshot.inclusion_block` was refuted (the register's L3 block -has the entry). Lesson recorded: when -deleting a mechanism, sweep for its *vocabulary* with review agents, not a -bare grep — one grep pipeline silently returned empty on files that -contained the pattern. - -All landed in the squashed branch commit; per-item dispositions in -[`register.md`](register.md). diff --git a/docs/review/2026-09-03-branch-stocktake.md b/docs/review/2026-09-03-branch-stocktake.md deleted file mode 100644 index 164797a1..00000000 --- a/docs/review/2026-09-03-branch-stocktake.md +++ /dev/null @@ -1,847 +0,0 @@ -# Branch stock-take (2026-09-03) - -Stock-take of the authority-boundary branch (PR #28, seven commits on -`f59ec25`) before it leaves draft, asked as: what is this branch for, is it -over-engineered, what next. Method: first-hand reads of the runtime, command, -recovery, lifecycle, and history code; then a read-only fleet of seven -subsystem lenses and five premise challengers (threat model, minimal design, -refuted-list audit, CI root cause, roadmap); then three adversarial refuters -per proposal for the eighteen highest-ranked proposals. Every proposal the -fleet raised is recorded here, including the ones that were not put to a -jury. Per-item dispositions that are actionable now live in -[`register.md`](register.md) (findings 19–31 and the 2026-09-03 refuted -block); this ledger is the full record and will be distilled when it closes. - -**How to read the status tags.** - -- **confirmed** — put to three refuters; at most one refuted it. The recorded - text includes the jury's amendments, which are load-bearing. -- **refuted** — put to three refuters; at least two refuted it on code truth - or on a registered invariant. The reason is recorded so it is not - re-proposed without new evidence. -- **unverified** — raised by one reviewer and not adversarially checked. - Treat as a reviewer's claim: re-verify the cited lines before acting. -- **verified first-hand** — checked directly against the tree during the - stock-take, independent of the fleet. - -## Landed - -Wave 1 (2026-09-03), one theme per commit, on top of the CI fix: - -- **CI red**: both wallet-sequencer binaries style logs only when stdout is - a terminal; the aging-tip scenario asserts exit code 10 instead of grepping - the rendered log (verified: the scenario passes and its log carries no - escape bytes). -- **Prose matches the types**: the token's true scope in `authorize()`, the - ADR, and the register; "boot-local" for the flush witness; the lock - witness's real predicate; three "journal" remnants; the error module's - dated history and removed command; a module doc that argued instead of - instructing; the recovery README's nonexistent type name; a test renamed to - what it asserts. Closes finding 25. -- **Codename sweep finished**: fifteen residual review codes replaced by - the reason or the invariant id. -- **Test-only storage surface gated**: `ensure_open_tip`, - `close_frame_and_batch`, `latest_batch_index`, `ordered_l2_txs_for_batch`, - `promote_finalized` are `#[cfg(test)] pub(crate)`, with their doc links and - the snapshot lifecycle doc pointed at the production paths. Closes findings - 17 and 21 and the second half of 10. -- **Two register nits**: the flusher's healthy retry logs at `warn!` - (finding 4); `fixed_mul`'s comment states what the truncation relies on - (finding 8's comment half). -- **`/tx` 500 body is fixed text**: the application's reason stays on the - lane error and the log (finding 5). -- **Panicking progress constructor deleted**: `ApplicationProgress::new` - had only test callers; `try_new` is the one constructor, tests use it with - `expect`. - -Wave 2 (2026-09-04), the containment diet, each commit refuted by three -read-only reviewers before landing: - -- **Containment writes nothing durable**: the in-scope fault recorder is - deleted; the command bracket's settlement write is the black box's one - writer, so a contained run records one row. The accepted loss (any death - before settlement leaves only the process logs) is stated in the runbook. - `run` logs the last black-box row once at startup, ahead of the preflight, - so the table has its first in-product reader. Closes finding 24. -- **Finalized lease is non-optional**: `acquire_finalized_lease` returns - `FinalizedLease { inclusion_block, dump }`; the impossible-`None` - containment branch in `finalized_state` is gone. Closes finding 26. -- **The reducer's one cycle is cut at the storage boundary**: the - `EnsureOpenTip` phase splits its guard (`TipAlreadyOpen`, retry) and - refuses inside its own transaction rather than commit without a Tip - (`TipMissingAfterOpen`, exit 30); `drive_recovery`'s doc records the - ≤5-phase bound. Closes findings 20 and 23. -- **Workers that only need to stop take the notification half**: the - detector, reader, and fee oracle take `ShutdownSignal` and hold their - own `ProcessLock`; the lane, server, and submitter keep the scope. Their - tests use a bare signal instead of a leaked-tempdir scope. The doc claims - that every worker held a scope clone are restated. Closes finding 22. - -Wave 3 (2026-09-04), the taxonomy, each commit refuted by three read-only -reviewers before landing: - -- **The key file classifies by kind**: a missing, unreadable, non-file, or - non-text batch-submitter key exits 30 like bad key content one call - later, instead of restart-looping at 1; environmental I/O still exits 1. - The typed `BootstrapError::KeyFile { path, source }` names the path and - never the contents. Closes finding 19. -- **Every recovery failure carries one verdict**: `RecoveryFailure::Provider` - is split into `ProviderUnreachable` (retry) and `SignerMisconfig` - (refuse), classified where it is built and pinned, payload and polarity, - against the `BootstrapError` projection. `classify_input_reader`'s doc - says what it can receive and why `Bootstrap` and `Join` flip polarity - between the startup phases and the live worker; both halves are pinned. - Closes finding 27; opens finding 32 (the same L1 misconfiguration exits 1 - under `setup` and 30 under `run`; recorded, not fixed). -- **Exit-code tests table-driven**: the five per-class tests, the verdict - test, the two startup-reader tests, the fee-oracle fatal-math test, and - the two app-bootstrap tests fold into five class functions and one test - asserting class and `is_terminal` per row (71 rows), with the five - verdicts pinned to the integers 10/20/30/40/1 through an exhaustive - match. Two wire-value pins outside the table: `run` on a never-set-up - data directory dispatches to 30 through the real command bracket, and - `stop_expecting_clean_exit` asserts SIGTERM→0 at the healthy stop of - `recovery_after_stale_batches`. Closes the SIGTERM→0 and 30-class halves - of the owed exit-code test. - -Wave 4 (2026-09-04/05), the documentation single-home passes. An -eight-topic read-only mapping fleet first found that the corpus already -elects its homes — the ADR for mechanisms 1, 2, and 4, the recovery README -for the reducer, I15 for the divergence freeze, the register for refuted -proposals, scheduler-semantics for the frame clock — so "the ADR becomes -pointers" was the wrong cut; the fan-out was in AGENTS.md, the check policy, -and the recovery README. Each batch refuted by three read-only reviewers -before landing: - -- **Only what the code enforces**: README's exit-code list gains 40 and - the SIGABRT class; the supervisor is "expected to honor" the contract, not - enforcing it; the lifecycle module doc says which commands preflight - `setup_complete` here and which admit through `commands::setup`; the - frontier writer's cutover forecast, the lane's vacuous "no L1 query", the - reducer's "exactly one phase", the watchdog notes' "structurally frozen" - frontier, the threat model's dangling "non-goals" pointer, and the check - policy's "today" all corrected; I15 gains the clean-exit re-check and - `LocalDivergenceFirst`; I9 owns the accepted false positive; ADR - mechanism 1 owns the hand-placed consult inventory, site by site, with - `runtime/shutdown.rs` and the register pointing at it. Three refuter - passes: the last two amendments of the consult wording were themselves - wrong until checked against a grep of every production consult site. -- **The register owns the rejected alternatives**: the ADR's list is six - names and a pointer; the register's block carries each argument, revisit - trigger, and Evidence line, gains the durable-phase-ledger entry it never - had and the cost datum the distillation dropped (stated as its source - states it, after two passes caught it overreaching), and records that the - boot-gate carve-out is now exercised. Mechanism 3 and G3 reduce to - pointers; the recovery README absorbs the loop, the four-fact inspection, - the witness erasure on Retry and Refuse, and the ≤5-phase bound. -- **AGENTS.md becomes a map**: the hot-path and storage sections are pointer - bullets plus the one rule nothing else owns; every clause only AGENTS.md - carried moved first (into ADR mechanism 4, I2, I3, I20, - scheduler-semantics' revisit trigger, the schema's identity and views - rules, README's closure rule, the design principles); the writer-role - table moves into `docs/invariants.md` corrected — one writer role per - fact, `deployment_identity` and the initial snapshot registration and the - anchor belong to setup, the watermark is shared under I14, startup - hygiene resets leases and collects dumps, the brackets write the black - box. The check policy's admission bullet and the recovery README's - divergence section keep what they own and point for the rest. -- **Six stub ledgers collapse** into the register's Review history table; - the two August ledgers and this stock-take stay for their evidence. The - marker-file containment protocol gets its refuted entry and the 2026-06 - "no architectural restructure" verdict its settled entry. -- **Module docs explain, they do not defend**; the abort bound's number - lives in `runtime/shutdown.rs` and the operator runbook only. - -Not yet landed: the PR title/body, which come last. - -## Verdict - -Proportionate overall, with three named pockets of residue. The maintainer's -fear was quantitatively wrong about scale and right about residue. - -| What | Lines | -|---|---| -| Branch diff | +20,459 / −5,252 across 127 files | -| Authority machinery changed (process lock, scope, lifecycle facts, supervisor) | ~1,700, under 900 production | -| Share of the branch | ~8% | -| Test functions | 418 → 588 | - -The lifecycle machinery that felt over-built was built and removed inside the -branch (decisions L1 → L2 → L3, ~490 production lines); what survived is three -admission facts and one append-only table. The heavy parts of the sequencer -are recovery and storage, which predate the branch and defend in-scope L1 -outage and zombie-transaction threats. - -The three pockets, in descending confidence: - -1. **The black box's write path.** `terminal_faults` has zero production - readers; the in-scope recorder opens a second SQLite writer from inside - containment, is the reason the "arm the watchdog before recording" ordering - hazard exists, and a contained run that drains normally writes two rows. - *(Landed 2026-09-04 as wave 2: the recorder is deleted, a contained run - writes one row, and `run` reads the table at startup — register finding - 24.)* -2. **The exit-code test encoding.** Sixty-seven hand-built projection asserts - pin a pure function four times over, while no test asserts that a real - failing process exits with the promised code; renumbering the terminal code - passes the whole suite. *(Landed 2026-09-04 as wave 3: the table, the - integer pins, and the two wire-value assertions; the stalled-safe-head - (class 20), 40, and 1 failure paths still assert only a non-zero status, - while the backward-clock-jump scenario already pins 20 and the aging-tip - scenario pins 10 — see the owed tests.)* -3. **Documentation fan-out.** Fact-derived admission is described in fourteen - places and the divergence freeze in eleven files; the branch's own L3 - rename missed three "journal" sites. - -Beyond mechanisms, four lenses independently found one defect class: prose -that claims more than the types enforce (see finding 25 in the register). - -**What passed the weight test and should not be cut:** the process lock; the -containment bit and the `Authorized` token at the ack, L1 send, and WS emit; -the pure reducer over one consistent inspection; the `RuntimeAdmission` -witness; the two-second abort watchdog (`/livez` returns 200 unconditionally, -so on a wedged post-containment drain nothing else pages); the clean-exit -divergence re-check. - -## Verified first-hand - -- CI red cause: `tracing-subscriber` 0.3.23 enables ANSI whenever `NO_COLOR` - is unset, with no TTY check (`fmt_layer.rs:743`); the harness inherits the - parent environment and pins only `RUST_LOG`; the e2e assertion at - `tests/e2e/src/test_cases.rs:3151-3157` greps for `status=TipInDanger(` and - the log carries `ESC[3mstatus ESC[0m ESC[2m= ESC[0m TipInDanger(0)`. It is - the only log-grep assertion in the suite. Both wallet-sequencer mains lack a - `with_ansi` call. -- PR metadata is stale: title "Fix storage decode policy", head branch - `feature/review-ledger-and-tracks`, body describing the decode-policy scope - and citing a retired codename; no risk/compatibility paragraph although the - baseline migration is rewritten in place and the `Application` hooks are - renamed. No reviewer comments. No `TODO`/`FIXME`/`unimplemented!` in the diff. -- `/livez` returns 200 unconditionally (`egress/api/health.rs:41-43`). -- `finalized_state` handles an impossible `None` on the `NOT NULL` - `inclusion_block` by escalating to containment (`egress/api/snapshot.rs:139-146`). -- Stale vocabulary: "journal" at `storage/history.rs:201`, - `commands/run/workers.rs:400` and `:680`; an "acknowledge" command at - `commands/error.rs:11`; `docs/recovery/README.md` names - `DangerDetectorExit::DangerDetected` (the type is `WorkerExit::DangerDetected`). -- `RecoveryProgress` derives `Copy`; `docs/recovery/README.md:315`, - `admission.tla:23`, and `recovery/mod.rs:893` call the witness "non-clone". -- `Authorized` is a real signature obligation at three functions - (`submit_batches`, `acknowledge_included`, `send_authorized`); it is minted - and discarded at `snapshot.rs:104/129/182`, `ingress/api.rs:87`, and - `inclusion_lane/mod.rs:211`; the batch-close and reconciliation commits at - `inclusion_lane/mod.rs:165/309` use the raw predicate. -- `terminal_faults` has zero production readers; a contained run appends two - rows (recorder raw cause, then the bracket's prefixed cause); - `latest_terminal_fault` returns the second. -- `ShutdownSignal` on main was 43 lines; `runtime/shutdown.rs` is now 450. - `http.rs` grew a lease-release supervisor with two containment call sites. - -## Confirmed (jury) - -- **cfg-test-gate-ensure-open-tip** (3–0). `Storage::ensure_open_tip` - (`storage/ingress.rs:104`) is `pub` with zero production callers — a new - instance of open finding 17 created by this branch, sitting beside its - guarded replacement. Gate it `#[cfg(test)] pub(crate)` (not private: eight - of nine callers live outside `storage::ingress`), de-link the two intra-doc - references at `ingress.rs:118` and `:486`, correct `:486`'s claim that the - runtime calls this form, and fix `docs/snapshots/lifecycle.md:26-31`, which - still credits it with the production genesis Tip. -- **stale-decision-carries-the-failed-condition** (3–0). - `ensure_open_tip_for_recovery` (`storage/recovery.rs:254-259`) raises - `StaleDecision { expected: Safe, actual: facts.danger }` for a disjunction, - so the `has_open_tip` case renders "expected Safe, found Safe", and - `recovery_tests.rs:114-132` pins that as intended. Add a payload-free - `RecoveryMutationError::TipAlreadyOpen`, split the two checks, add a paired - `RecoveryRetryReason::TipAlreadyOpen` so `classify_mutation` (which today - discards `expected`) carries it to the operator, classify Retry, update the - test and the polarity pin. Roughly +14/−6 across three files. The arm is - production-unreachable under the process lock; this is diagnostics. -- **detector-takes-shutdown-signal-not-runtime-scope** (2–1). - `DangerDetector` and `InputReader` use the scope for exactly one thing, - `wait_for_shutdown`, and each already carries a construction-required - `ProcessLock`. Narrow `start`/`start_preflighted`/`run_forever` to - `ShutdownSignal` and pass `scope.signal()` at the two launch sites. The - change is incomplete without restating three doc comments that assert the - property it relocates: `workers.rs:100-105` and `:1120-1124` ("every spawned - worker retains a RuntimeScope clone, which also retains the process lock") - and `shutdown.rs:96-99` ("workers that touch the data directory take a - scope", already false for the submitter). Restate as: workers that - externalize or contain take a scope; data-directory ownership is a separate - construction-required `ProcessLock`. The dissent would narrow only the - detector. -- **app-with-progress-wrapper** (2–1, a do-not-adopt). Moving - `ApplicationProgress` into a sequencer-owned wrapper (deleting both - capabilities, the seal, three trait methods, all three asserts, ~130 lines) - is not viable: the pair is inside the canonical SSZ bytes - (`examples/app-core/src/wallet_snapshot.rs:41-42`) that `create_dump` writes, - `/finalized_state` streams, and the watchdog byte-compares, and the canonical - machine advances it inside its own state transition; cockroach recovery reads - the clock out of a dump into a wiped database (`commands/setup/mod.rs:433-451`, - `Checkpoint::load`). The rationale for the clock exists at - `docs/snapshots/format.md:113-118`; the missing piece is the composition. - Add one sentence to `docs/protocol/application-contract.md` §4 and to - `ApplicationProgress`'s doc comment, cross-referencing `format.md`, and - extend `format.md`'s "must live in the canonical state bytes" sentence to - cover `executed_input_count`. -- **bound-or-prove-drive-recovery-termination** (2–1). `drive_recovery` - (`recovery/mod.rs:288-313`) is an unbounded loop with one cycle: - `Repaired` + `Safe` + `!has_open_tip` → `EnsureOpenTip` → `Repaired`. No - watchdog exists on the boot path (the scope is constructed in `prepare`, - after recovery). Main could not spin. Take the postcondition, not the loop - bound: after `open_fresh_tip_in_tx` in `ensure_open_tip_for_recovery`, - re-read `has_valid_open_batch` and return a new typed variant classified - `RecoveryError::refuse` (exit 30) — not a `debug_assert` (compiles out in - release; the file's existing postcondition at `ingress.rs:538-541` is one), - and not `StaleDecision` (maps to retry and relocates the non-termination into - the supervisor). Record the ≤5-phase bound in `drive_recovery`'s doc. - Alternative accepted by two jurors: make the reducer's `Repaired`+`Safe`+no-tip - arm a terminal `Refuse`, removing the cycle from the pure function. The - dissent notes the antecedent is unreachable by SQLite semantics; the - counter-argument that carries is fidelity — `admission.tla:271-286` - hardcodes `hasOpenTip' = TRUE`, so TLC proves termination of a model whose - postcondition the code does not enforce. -- **typed-key-source-io-error** (2–1). `resolve_key_source` - (`commands/config.rs:243-254`) returns a bare `std::io::Error` that lands in - `CommandError::Io` → exit 1 ("restart with backoff") for a missing or - unreadable key file, while bad key content in the same file exits 30 via - `SignerMisconfig`. Kind-filter rather than blanket-map, mirroring - `referenced_artifact_io_is_terminal` (`dump_info.rs:49-58`): NotFound, - PermissionDenied, InvalidData, IsADirectory, NotADirectory terminal; - everything else operational (a not-yet-mounted secret must not consume the - do-not-restart code). Prefer a distinct `BootstrapError::KeySourceUnreadable - { path, kind }` over reusing `SignerMisconfig`; never echo file contents. - Roughly +30–40 lines with the predicate and tests. Consider the same - treatment for `create_dir_all` or record why not. -- **table-drive-exit-code-tests** (2–1). Fold the five per-class tests and - the duplicating verdict test (`error.rs:782-828`) into one `const CASES` - table keeping every distinct error shape and every reason string as the - assert message; drop the verdict column (derivable from the bijection); - assert `is_terminal()` per row, which extends a five-shape pin to ~54. Do not - invent rationales for rows that carry none. Realistic saving is 110–150 - lines, not 250. The point is the spend: no test asserts a real failing - process's exit code (the four failure-path e2es assert only `!success()`), - and the `EXIT_*` values appear as literals only at their declarations, so - renumbering `EXIT_TERMINAL` to 31 passes the suite. Add SIGTERM → 0 (the - harness already waits and discards the status) and one 30-class failure → 30 - (`run` on a never-set-up data directory needs no new lever), asserting integer - literals. Correction from the jury: composition is pinned in-crate at - `workers.rs:1209/1228`, `run/mod.rs:237`, `startup_hygiene.rs:168/191`, - `commands/mod.rs:330`, `process_lock.rs:159`; the gap is the process-level - projection at `harness.rs:117-119`. -- **dedupe-terminal-fault-rows** (2–1 for documenting; 0–3 against skipping - by variant). Document the two-row shape at `record_terminal_fault`, in the - ADR's black-box paragraph, and in the runbook's postmortem line, including - the asymmetry: a clean contained drain yields two rows; a controller panic or - watchdog abort yields one. Do not skip the bracket write when the error is - `StorageInvariantViolation`: the recorder swallows both `open_writer` and - `record_terminal_fault` failures into a `warn!`, so the variant does not - prove a row landed, and the post-drain bracket write is the attempt more - likely to succeed after `SQLITE_FULL` or contention (5 s `busy_timeout` - against a 2 s abort deadline). If one row per fault is wanted later, - condition the skip on evidence (an `AtomicBool` the recorder sets on - success), not on the variant. - -## Refuted (jury) — do not re-propose without new evidence - -- **supervise-workers-with-joinset** (3–0). `select_first_exit` and `finish` - deliberately read the same worker return two ways: `WorkerStop::from_select` - maps `Ok(Ok(()))` to `StoppedUnexpectedly` (the runtime is live), while - `from_shutdown` maps it to `Ok(())`. Feeding both phases from one `JoinSet` - of `wait_for_*_shutdown` futures collapses them into the shutdown reading, so - a worker that dies silently while live yields a value `FirstExit` cannot - represent; the available completions are "run with a dead lane" or "drain - and exit 0", the one code `run/mod.rs:82-87` names as breaking the - supervisor's rediscovery chain. `into_supervision(self)` would also drop - `ShutdownOnDrop`, requesting shutdown milliseconds after launch; the - conditional fee-oracle push relocates rather than deletes; and - `FirstExit::detector` plus its mapping tests disappear (one of the four - reasons the register already refuted this shape on 2026-08-23). **Survives:** - the `swap_remove` hazard at `workers.rs:653-655` is real and unwritten in - types; a cleanup-only `JoinSet` built inside `finish`, with the live race - untouched, was not what the jury examined. -- **collapse-preparedruntime-into-boot** (3–0). The load-bearing claim ("zero - tasks during fallible work is reviewer-visible, not type-enforced") is false: - `fn launch(self, _admission: RuntimeAdmission) -> Workers` (`workers.rs:288`) - is non-async and non-`Result`, so a `?` or `.await` between admission and the - six spawns is a compile error today. `async fn boot(..) -> Result<..>` makes - both silently legal, reopening a guarantee registered in the check policy, - ADR mechanism 1, and AGENTS.md, and the linearization argument at - `recovery/mod.rs:477-483`. Under `boot` nothing consumes `RuntimeAdmission` - (`let _a = admit_runtime()?` satisfies `#[must_use]`). Commit 02a2b34 ran - this pass and deliberately stopped here. **Survives:** the test - `preparation_outliving_clean_facts_cannot_launch` (`workers.rs:1161`) never - calls `launch`; rename it to what it asserts. -- **make-authorized-token-uniform** (3–0). `LeasedDumpBody` does not exist - (the primitive is `stream_body(file, guard)` at `snapshot.rs:214`); - `finalized_inclusion_block` (`snapshot.rs:101-120`) has no streaming - primitive to receive a token; and `ingress/api.rs:87` is not a pre-check — - its comment calls it the publication gate, it runs after the lane's ack - resolves, and it immediately precedes the success body that is the soft - confirmation leaving the process, the ack family the ADR names as a token - site. Only the `inclusion_lane/mod.rs:211` half survives (a fast-turn entry - gate; the real ack boundary re-consults at `:248`). **Survives:** the doc - tightening — the token proves "consulted at some point in this borrow", not - "at this effect boundary" (the poster mints at `worker.rs:243` and then does - a chain-id RPC, fee estimation, and a nonce fetch before its own re-checks); - and the coverage claim in the ADR/register should say three compile-forced - primitives plus hand-placed consults at the HTTP 200 gate and the two lane - mutation commits, or the 200 body should take the token (~8 lines). -- **recovery-polarity-unconstructible** (3–0). Diagnosis exact: - `RecoveryError::retry(RecoveryRefusalReason::CanonicalDivergence{..})` - compiles today and would project the absorbing refusal to exit 20 (bounded to - one restart by the next boot's preflight). But `recovery` is `pub mod` and - both enums have public variants, so deleting the `#[from]` impls removes the - shortest spelling, not the route: `RecoveryError::retry(RecoveryFailure::PolicyRefusal(r))` - still compiles, and that longer spelling is the dominant idiom at all 15 call - sites. Cost ~16 renames for a property not achieved. The invoked precedent - (1fcb9aa) deleted the violating value from the type; this does not. -- **single-table-per-error-type** (3–0). `From - for BootstrapError` (`error.rs:671-683`) performs no terminal/transient - decision — it selects among three variants with distinct fields, and the - verdict is taken later over the `BootstrapError` taxonomy, whose variants - have four other producers. "Have both sites read one `is_terminal`" is not - implementable; the result is a third table. Part (b)'s premise is false: - `reader.rs:96-104` states the phase-dependence for `Bootstrap` and `Join`. - Renaming to `is_terminal_in_worker` is wrong for `FlushError` (no - `WorkerExit` arm), and "phase in the type" is blocked because the phase - belongs to the caller (`create_provider(..).map_err(InputReaderError::Bootstrap)` - appears identically in `sync_to_current_safe_head` and `run_loop`). - **Survives:** `classify_input_reader` (`recovery/mod.rs:526`) carries no doc - comment and its `Bootstrap`/`Join` refusals are pinned by no test; and a - pre-v3 InputBox exits 1 under `setup` but 30 under `run` — a separate - finding. -- **flatten-recovery-error-to-one-enum-with-is-retryable** (3–0). A flat - `is_retryable(&self)` must be total over the value, and one variant carries - two verdicts: `ProductionRecoveryDriver::flush` (`recovery/mod.rs:429-438`) - maps `VerifiedSignerProviderError::ChainIdRpc` → retry and `::Create` → - refuse into the same `RecoveryFailure::Provider(String)`. Either resolution - regresses (a bad RPC URL restart-loops forever, or a transient chain-id - timeout pages), nothing pins either arm, and dropping the `Box` risks the - deliberately managed `CommandError` footprint under `result_large_err`. - **Survives:** split `Provider(String)` into two verdict-determined variants - as an independent fix; the wrapper's doc claims a context-sensitivity the - `classify_*` functions do not use. -- **drive-recovery-owns-phase-to-progress-mapping** (2–1). The replacement - is not total: `(RecoveryPhase::Flush, PhaseOutcome::Done)` has no target - because `Flushed { observed_safe_block }` needs a block number the loop does - not hold; the pre-existing implementation of exactly this mapping - (`PhaseCompletion` + `transition_after_phase`) was deleted by `ed41f9b`, whose - message pre-answers the argument. **Survives:** delete - `RecoveryDriver::admitted` (`recovery/mod.rs:283`, a production trait method - whose only implementor pushes a string into a test trace; `drive_recovery` - returns `Ok(())` only from the Admit arm); and the five trace tests exercise - the double's copy of the mapping while production's copy is pinned by no unit - test. -- **merge-stringly-bootstrap-variants** (2–1). `FeeOracleMisconfig` has two - further producers (`setup/mod.rs:144` and `:171`) passing bare strings whose - "fee oracle misconfiguration" words exist only in the variant's Display, and - the black box stores `error.to_string()`, so merging attributes an operator's - Uniswap mistake to a trusted-code fault in the one postmortem artifact. Part - (b) demotes a compile-forced classification to an unchecked `&'static str` - discriminant, the shape the register refuted twice on 2026-08-23. Real delta - ~−15, not −25. -- **terminality-trait-for-workerstop** (3–0). Inverts its goal: - `WorkerExit::is_terminal` already matches all seven variants by name; the - trait admits `fn is_terminal_invariant(&self) -> bool { false }` exactly as - plausibly as `|_| false`. `impl TerminalityOf for std::io::Error { false }` - installs a crate-wide answer for a type the codebase has decided has no - context-free answer (`dump_info.rs:49-58` classifies several kinds terminal); - a `pub(crate)` trait in a public type's bound trips `private_bounds` under - `-D warnings`; and `is_terminal_invariant` is inherent on eight types, only - five of them `WorkerExit` payloads. Delta inverts to +4..+15. -- **prune-duplicate-tla-actions-and-run-check-admission-in-ci** (3–0). - "Nix already provides tlc" is false for CI: no Nix expression or `.envrc` is - tracked, `ci.yml` provisions tools by hand, and `just` is absent from the - `rust` job; TLC also checks the spec against itself and says nothing about - spec-vs-Rust drift. The three deletions are state-space-neutral (`Crash` - subsumes every settle action), but the rationale is false: `decision` - records Retry/Refuse, `DecideRetry`/`DecideRefuse` have distinct guards, and - `InspectRetry` encodes its own comment ("a known local divergence cannot be - masked by the retry edge"). **Survives:** put the 860-state model under CI as - a properly pinned standalone `formal` job (JDK + `tla2tools.jar` pinned by - version and sha256 in `toolchain-pins.env`). - -## Unverified (raised by one reviewer, not put to a jury) - -Re-verify the cited lines before acting on any item below. - -### Runtime authority and containment - -- **drop-in-containment-fault-recorder** (threat challenge, Δ−110). Delete - the `FaultRecorder` alias, field, `set_fault_recorder`, and the recorder - invocation from `runtime/shutdown.rs`; delete `install_terminal_fault_recorder` - from `workers.rs`. Containment becomes three non-blocking steps (set the - cause, arm the watchdog, request shutdown), removing the "either may block" - ordering hazard and the second SQLite writer inside containment. The bracket - write at `run/mod.rs:90` keeps recording every contained fault that settles. - Loss: a fault whose drain hangs past 2 s and exits via SIGABRT leaves no row - — already the documented status for unclean deaths - (`docs/watchdog/operator-deployment.md:392-395`). Note the confirmed - dedupe verdict above: the bracket write is a genuine retry, which argues for - keeping one writer rather than two, and for the bracket one. *(Landed - 2026-09-04, wave 2; register finding 24.)* -- **delete-terminal-faults-black-box** (threat challenge, Δ−330) and - **cut_black_box** (minimal design, Δ−185). Delete the table, its two - triggers, `TerminalFault`, `record_terminal_fault`, `latest_terminal_fault`, - `LifecycleCommand::parse`, `record_terminal_fault_best_effort` and its four - call sites; replace the runbook's `SELECT * FROM terminal_faults` paragraph - with the log-and-exit-code instruction. New evidence against decision L3: - zero production readers; no CLI or API surface; the write path spans four - files; cockroach recovery wipes the database in exactly the incident class - where the postmortem matters. Counter-argument neither author could dismiss: - a Kubernetes Deployment restarts regardless of exit code, so a terminal - fault restart-loops and the first cause could rotate out of the logs while - the black box retains it. Judgment call, not a defect. -- **close-or-correct-the-token-coverage-claim** (threat challenge, Δ+8). - Either make `ingress/api.rs:85-92`'s success response take `Authorized` - (mirroring `acknowledge_included`), or amend ADR mechanism 1 and the - register's settled entry to the true scope. Do not extend the token to the - lane's mutation commits (they sit inside `&mut self.storage` borrows). The - jury's refutation of the "uniform" proposal above endorses this framing. - *(Resolved 2026-09-03 by restating the ADR and the register to the true - scope; register finding 25.)* -- **sketch_boot_shutdown** (minimal design, Δ−260) — a reference sketch that - keeps the lock, scope, token, `ShutdownOnDrop`, reducer, hygiene, lifecycle - facts, and the typed `Workers`/`FirstExit`, and deletes `ShutdownSignal` - (fold into the scope), `RuntimeAdmission`, `PreparedRuntime`/`WorkersConfig`, - and the `WorkerId`/`ComponentShutdown`/`next_component_shutdown`/six-waiter - drain in favour of `Option` fields taken at the winning select - arm and one `tokio::join!` over a generic `drain`. Partly overtaken: the - jury refuted the `PreparedRuntime` collapse and the `ShutdownSignal` half is - contradicted by the confirmed narrowing (the slim half gains two consumers). - The `Option`-take drain is the one part not examined by a jury; it avoids - all four grounds of the 2026-08-23 refutation (named fields, named arms, no - `swap_remove`, no empty-list race). -- **cut_admit_runtime** (minimal design, Δ−60). Delete `RuntimeAdmission`, - `admit_runtime`, `AdmissionChanged`, and the `launch(_admission)` parameter; - keep the fallible-then-infallible ordering. Argument: between the reducer's - `Admit` and launch, the lock excludes every other process and zero tasks - exist, so only wall-clock drift can change the facts, and those arms are the - Retry class re-derived by the detector within one 2 s poll. Adjacent - refutation applies in part: the jury defended the witness as the thing that - makes "launch only from a fresh admit" a compile fact and the basis of the - linearization at `recovery/mod.rs:477-483`. Low priority. -- **taxonomy_min** (minimal design, Δ−145). Regroup `BootstrapError`'s - seventeen variants into verdict-uniform groups (`Misconfig(..)` uniformly 30, - `Transient(..)` uniformly 20, `Recovery`, `SetupNotComplete`, `SetupRefuse`, - `OpenStorage`), move `IdentityError::FirstBootRequiresL1` into the transient - group so `IdentityError` becomes uniformly terminal, delete - `CommandFailureVerdict` (five variants in bijection with five constants, two - consumers), delete `WorkerId` if the drain no longer needs identity. The - taxonomy has exactly one consumer outside the crate (`harness.rs:117`). - Partly overtaken: the jury upheld keeping `is_terminal()` as the thing the - black-box write gates on. -- **leased_dump_inclusion_block_non_optional** (refuted-list audit, Δ−4; - premise verified first-hand). Stop sharing `LeasedDump` between the - finalized and latest lease queries (own return type or `LeasedDump`), and - delete the `let Some(inclusion_block) = leased.inclusion_block else { - contain(..) }` branch at `snapshot.rs:139-146`. The column is `NOT NULL` - (`0001_schema.sql:855-856`); the L3 review refuted a boot assert on it and - left a heavier runtime branch that maps a type artifact to exit 30, contrary - to the check policy's "no `Option`-handling for can't-be-`None`". -- **startup_log_last_terminal_fault** (refuted-list audit, Δ+8). After the - preflight in `run`, read `latest_terminal_fault` and emit one `warn!` when - present. Explicitly not a gate: no acknowledgement, no branch on the value. - The recorded refutation argues against a gate on a verdict; a read that - changes no decision is untouched by it. This would give the black box its - first in-product reader; if declined, say in the register that the black - box is an out-of-process artifact by design. *(Landed 2026-09-04 as - `warn_on_previous_terminal_fault`, ahead of the preflight; register - finding 24.)* -- **startup_hygiene_single_finalized_read** (refuted-list audit, Δ−6). - `require_finalized_snapshot` and `restamp_finalized_promotion` each query - `finalized_dump()`; fetch once in `run_snapshot_hygiene` and pass the row. - Ordering of the five steps is unchanged. -- **reconsider-release-supervisor-weight** (lane lens, ~175 lines). The - supervised lease-release queue in `http.rs:157-235,325` (unbounded MPSC of - boxed closures, a `JoinSet` supervisor with a two-armed `select!`, a drain - awaited inside `axum::serve`, containment on both joins, `ReleaseScheduler` - changed to `Arc` plus a second reporter) defends a real but narrow - hole: a `StatementChangedRows` on release means the leased row vanished, - which nothing else re-detects. Two lighter shapes: (a) keep the - classification, drop the supervisor, accept that a release racing the very - end of shutdown may miss classification (~−120 lines); (b) keep the drain - but move it to the egress snapshot module that owns leases, so `http.rs` - stops hosting a runtime component. The queue is unbounded and bounded only - by concurrent snapshot requests, which have no cap. -- **drop-token-ceremony-at-bool-sites** (lane lens, Δ−10). Overlaps the - refuted uniform-token proposal: the `/tx` publication-gate half is refuted - (it is the ack); the snapshot half survives only as "push the token into - `stream_body`" for the two streaming routes. Separately worth a maintainer - decision: the `/tx` gate returns 503 for an operation that is durably - committed and may still reach L1; the API contract should state the - client-visible semantics of "503 after commit". - -### Error taxonomy - -- **fee_price_stamp_surface_or_drop** (refuted-list audit, Δ+10). - `log_gas_price_updated_at_ms` is written on every refresh - (`storage/fee_oracle.rs:29-40`) and read only by tests, yet the threat model - cites it as the honest telemetry that justifies having no expiry gate. - Either surface the age in `GET /healthz` as an informational field, or - include `retained_price_age_ms` in the existing transient-refresh warn, or - drop the column and fix the threat-model sentence. No threshold, refusal, or - lifecycle effect is proposed. -- **register_provenance_and_wording** (refuted-list audit, Δ+6). Applied in - the register on 2026-09-03: the fee-price-age refutation was added by - 143a290 (2026-08-25) but filed under "From the ADR re-evaluation - (2026-08-01/02)"; the boot-gate refutation's "(in any form)" is broader than - the argument it rests on; the drain-merge entry says "Scope-narrowed - 2026-08-23" while the landing commit is dated 2026-08-24. - -### History foundation and the application boundary - -- **drop-panicking-progress-constructor** (Δ−12). `ApplicationProgress::new` - panics on an incoherent pair and has only test callers; its own doc says to - use `try_new` on the only path that constructs one from data. Delete it; - tests become `try_new(..).expect(..)`. Register finding 17's category. -- **defer-era-newtypes-to-track3** (Δ−110). The schema slice (`history_state` - columns, five triggers, the generation bump inside `cascade_and_reopen`) is - cheap to carry and expensive to retrofit, and should stay. The Rust surface - (`EraId` with Display/Debug/TryFrom, a three-variant parse error, - `RecoveryGeneration`, `HistoryVersion`) has zero production readers; the WS - feed destructures the coordinate away with `..` (`l2_tx_feed/mod.rs:299-315`), - and Track 3 says the era leg is explicitly unconfirmed by the consumer. - Alternative: represent the era as `[u8; 16]` at the storage boundary and let - Track 3 introduce the newtypes beside the wire codec. Low cost either way. -- **drop-uuid-version-variant-checks** (Δ−30). `mint_era_id` - (`open.rs:242-261`) stamps v4/RFC-4122 bits into a random blob, then the Rust - constructor and a SQL `CHECK` verify that self-imposed constant at three - points, for a token whose only semantics is equality. Keep the 16-byte - newtype, length check, and hyphenated Display; drop the version/variant - stamping and checks (+6 bits of entropy). The one argument for keeping it is - a future strict-UUID consumer, a Track 3 wire concern; if kept, reword the doc - from "must carry" to "presentational contract for the future wire form". -- **single-enforcement-for-mapping-contiguity** and - **drop-duplicate-offset-assert** (Δ−15). `attach_executed_inputs_in` - (`history.rs:116-133`) recomputes the exact predicate - `trg_executed_inputs_contiguous` (`0001_schema.sql:562-575`) enforces, on the - accepted user-op path with the latency contract — one `query_history_state` - read plus one `MAX(executed_input_offset)` probe per chunk. Its own comment - says the schema independently enforces the rule. Keep one enforcement point, - preferably the trigger (cannot be bypassed by any writer, aborts the - transaction rather than unwinding a panic through the lane). For directs the - offset is checked a third time by the derive-and-compare below. -- **drop-terminal-fault-typed-reader** (Δ−55). `latest_terminal_fault`, - `TerminalFault`, `LifecycleCommand::parse`, and the two `Malformed` variants - that only report a malformed black-box row exist to serve three test - assertions; an empty cause is already impossible at the engine - (`0001_schema.sql:628-630`). Contradicted in part by - `startup_log_last_terminal_fault` above, which would give the reader a - production caller; decide the black box's reader story once. *(Withdrawn - 2026-09-04: the startup read landed, so the typed reader has its - production caller.)* -- **single-admission-implementation-for-setup** (Δ−25). - `preflight_lifecycle_command` has two callers (`run`, `flush`); setup and - rebuild go through `admit_setup_lifecycle` (`setup/mod.rs:361-388`), which - re-implements the same two facts with different semantics (an - already-complete plain setup is a no-op success there, `NotAdmissible` in - the lifecycle module). Make `preflight_lifecycle_command` return a three-way - admission for setup/rebuild and have setup call it. Also `run` calls the - preflight (which refuses without `setup_complete`) and then - `load_setup_identity` re-checks completion with a different error type; drop - the second check. - -### Lane and storage - -- **narrow-direct-attribution-cross-check** (Δ−8). `persist_frame_direct_sequence` - (`mutations.rs:168-181`) re-derives every direct's sender over the whole - drained range and asserts vector equality with the lane's receipts inside the - reconciliation commit; the lane read the same rows moments earlier. Cheaper - shapes: carry the skipped-submitter count and assert - `executions.len() + skipped == range.len()` plus first/last offset, or keep - the derive behind `cfg(debug_assertions)`. The honest answer depends on the - catch-up ACK measurement the register already owes (5,000 directs over a - 7,200-block jump in one turn). - -### Tests and e2e - -- **gate-remaining-test-only-storage-api** (Δ−40). `latest_batch_index` - (`l1_submission.rs:97`), `ordered_l2_txs_for_batch` (`:129`), and - `promote_finalized` (`snapshot_dumps.rs:182`) are `pub` with only test - callers; `promote_finalized` can promote without the lane's inclusion-block - and lease invariants. Delete the first two (fold into their tests), gate the - third. This is what the in-crate test move was supposed to unlock (register - finding 17). -- **drop-tryfrom-accepts-tests** (Δ−35). Five `*_accepts_*` tests in - `storage/convert.rs` assert that `std::convert::TryFrom` is correct; keep - every `should_panic` twin (they pin the settled decode policy) and - `prepare_time_sql_failures_classify_persistent_in_both_spellings`. Also - `era_id_displays_canonical_lowercase_hyphenated_form` pins a Display string no - consumer parses. Record in the register that the 21 new - `#[should_panic(expected = ..)]` attributes are accepted panic-message - coupling. -- **unify-harness-chain-clock** (Δ+60). Four notions of block time exist: - `SECONDS_PER_BLOCK = 12` duplicated at `rollups.rs:264` and - `sequencer.rs:876`, `LIVE_L1_BLOCK_INTERVAL_SECONDS = 1`, `BOOT_L1_MINE_INTERVAL - = 1 s`, and the sequencer's configured `seconds_per_block = 12`; e2e - correctness depends on their unwritten relationship staying under the 12 s - clock-usability threshold. `advance_live_frame_until_covers` - (`test_cases.rs:480-514`) can drive L1 roughly 20× ahead of the process - clock per iteration. Give the harness one `ChainClock` owned by the devnet - stack, constructed from the value passed to the sequencer, with - `advance(Duration)` and `mine_live(n)` deriving from it, and one post-mining - check `l1_head_timestamp − faketime_now < seconds_per_block` so drift fails - loudly in the harness. Two of the three re-staged scenarios are principled - and strictly stronger (`sequencer_outage_danger_zone_tip_cascade` now asserts - an invalidation; `wall_clock_backward_jump_retries_then_recovers` is the only - per-variant exit-code e2e in the suite). -- **replace-timewarp-tip-injection** (Δ−10). `aging_open_tip_runtime_danger_zone_exit_test` - injects a wedged lane by mining 1,150 blocks with wall time frozen (a chain - 3.8 h in the future), then compensates with `mine_live_l1_blocks(1)` plus an - absolute faketime offset, and greps the log to prove the future-dated view did - not route into the clock-fallback arm. Replace the injection with a - lane-level one (a `--freeze-frame-clock` test dial beside the existing - batch-open dial), advance wall and L1 together, and delete the compensation - and the log assertion. Minimum fix: assert exit code 10 instead of the log. - Also `set_faketime_offset` resets the cumulative counter while leaving an - absolute offset in the rc file, so a later `advance_wall_and_mine` in the - same scenario would regress the child clock. -- **replace-watchdog-sleep-assertions** (Δ−20). Two watchdog tests are - negative assertions implemented as `recv_timeout(250 ms).is_err()`; - unfalsifiable by slowness. Expose `is_watchdog_armed()` under `#[cfg(test)]` - or have the injected abort action record whether a deadline was scheduled. -- **merge-detector-arm-mapping-tests** (Δ−25). Three tests cover the - 13-line `FirstExit::detector`; merge into one table test over the four join - shapes. The composed containment tests the 2026-08-23 refutation protects are - untouched. -- **document_second_half_clock_assumptions** (Δ+12). Record at - `test_cases.rs:3168-3176` that the `+1` alignment margin is not the real - margin (Anvil block timestamps track wall time) and that the single - `mine_live_l1_blocks(1)` refresh has one block of headroom only because Anvil - runs with `--slots-in-an-epoch 1`. -- Also open: `RuntimeScope::default()` (`shutdown.rs:258-267`) leaks one temp - directory per construction via `mem::forget`; worth asking whether a - `(RuntimeScope, TempDir)` guard should be the only shape. - -### Documentation corpus - -- **refuted-evidence-grades** (Δ+10). Give each refuted entry an `evidence:` - line naming the file/line or measurement a reader can re-run; demote entries - that cannot produce one to "declined, no evidence recorded". Restore the - deleted cost datum to the per-chunk divergence-query entry (the - pre-distillation ADR read "every roughly 14-ms user-op chunk"; "14 ms" - appears in zero markdown files now). Name the select arm in the - homogeneous-list entry's title, since the `Vec<(WorkerId, ComponentShutdown)>` - shape now exists in the tree for cleanup. *(Partly landed 2026-09-05: - the ADR-list block carries Evidence lines and the cost datum is restored - as its source states it; the other refuted blocks and the select-arm - title remain.)* -- **collapse-six-stubs** (Δ−110, −6 files). Six of the eight dated ledgers are - 15–20 line stubs carrying a verdict plus a pointer; collapse them into a - "Review history" table at the bottom of the register. Keep the two August - ledgers (they carry the only re-verifiable evidence in the corpus). - *(Landed 2026-09-05; this stock-take stays too, as the branch's ledger. - The six files totalled 106 lines, not 110.)* -- **adr-dedupe-vs-register-and-invariants** (208 → ~70 lines). Each ADR - mechanism is also described in the invariants check policy, AGENTS.md, the - recovery README, the threat model, the runbook, and module docs; five of six - rejected alternatives are also in the register's refuted list, and the two - point at each other circularly. Cut the ADR to context, the policy statement, - and four mechanism names with pointers; move the rejected-alternatives - arguments into the register so there is one home. *(Landed 2026-09-05 - with a correction: the mapping pass found the corpus already elects the - ADR as the home for mechanisms 1, 2, and 4, so the ADR keeps those and - points for mechanism 3, G3, and the rejected list; the register owns the - arguments.)* -- **single-home-divergence-freeze** (Δ−60). `docs/invariants.md:353-372` and - `docs/recovery/README.md:398-415` are the same four sentences; the ADR's G3 - and `AGENTS.md:264` are third and fourth compressions. I15 owns the runtime - reaction and race bound; the others link. *(Landed 2026-09-05.)* -- **agents-hotpath-to-pointers** (50 → ~20 lines). `AGENTS.md:255-304` - restates I2, I3, I9/I15, I17, I18, and the admission policy in fifteen - paragraph-length bullets, violating its own line-475 rule; the good pattern - is already used at `AGENTS.md:118-121` and `:324`. *(Landed 2026-09-05, - with the storage section and the writer table, which moved to - `docs/invariants.md` corrected.)* -- **module-docs-explain-not-defend** (Δ−15). Strike the four defensive - clauses (`workers.rs:21-27` "they are the enforcement, not style"; - `error.rs:10-13`'s dated `RunError` history and stale "acknowledge"; - `shutdown.rs:20-23`; `storage/recovery.rs:15-23`), and fix the three "journal" - usages. *(Landed 2026-09-05; the "journal" usages went in wave 1.)* -- **finish-the-codename-sweep** (~14 one-line edits). Residue: `history.rs:202` - ("L2"), `l1_inputs.rs:41` ("H6"), `wallet.rs:101` ("D10") added by this - branch; `provider.rs:160,234`, `e2e_sequencer.rs:411,429,537`, - `tests/harness/src/sequencer.rs:33,38,302,305,583`, `test_cases.rs:3039,3924` - predate it. Track 6's requirement labels R1–R5 collide with the codename - map's R1–R5. *(Landed in wave 1; a 2026-09-04 re-check of the residue - list found only interval notation left.)* -- **proportionality-measured**. Measured: ~8,026 lines of standalone doc/spec, - ~6,792 comment lines, ~21,200 lines of production Rust, roughly 0.7 prose - lines per code line; the branch's own margin is one doc line per six code - lines. Volume is defensible; the unstated fan-out is not. Either adopt a - single-home rule with a named canonical copy per mechanism, or write down - that redundancy is deliberate and name the canonical copy. *(Decided - 2026-09-05: the single-home rule, with the homes named, is a settled - register entry.)* -- Also open: `docs/plans/` is listed as timeless in AGENTS.md but the tracks - board is a dated status board; the deleted terminal-containment plan's - marker-file protocol has no refuted entry anywhere. *(The marker-file - entry landed 2026-09-05; the `docs/plans/` question stays open.)* - -### CI - -- **binary_disables_ansi_when_not_a_tty** (Δ+4, rank 1). Add - `.with_ansi(std::io::stdout().is_terminal())` to both wallet-sequencer mains - (`IsTerminal` is std). Independent of the test: a daemon writing to a pipe, - file, or journald must not emit SGR escapes. -- **assert_exit_class_not_log_text** (Δ−4, rank 2). Replace the log grep with - `exit.code() == Some(10)`; `TipInDanger` projects to 10 and the clock - fallback to 20. Caveat: 10 does not separate `TipInDanger` from - `ClosedBatchInDanger`; if that matters, keep one check anchored on the - never-styled substring `TipInDanger(` alone. -- **harness_pins_no_color** (Δ+2, rank 3). Pin `NO_COLOR=1` beside the - `RUST_LOG` pins at `sequencer.rs:1300` and `:1394` as hermeticity, not as the - fix. -- **strip_ansi_in_assertion** — rejected by its author; recorded so it is not - re-proposed. -- **centralize_tracing_init_in_run_main** (Δ−10, optional). The two mains are - byte-identical apart from the config constructor; `run_main` already owns the - exit-code contract and could own log rendering. A library installing a - global subscriber is a deliberate boundary decision, not part of the CI fix. - -### Roadmap items (plan, not simplification) - -- **pr-body-rewrite** — a ~270-word draft exists in the fleet output; the - title should name the actual scope and the body must carry the two breaking - changes (baseline migration rewritten in place; `Application` hooks renamed). -- **pre-draft-ci-fix**, **pre-draft-metadata** — the two blockers; plus - `docs/plans/2026-07-coordination-tracks.md`'s "ready for its PR against - main" line and the register's verification date. -- **fold-before-merge** — findings 4 (flusher `error!` → `warn!`), 5 (`/tx` - 500 body echoing `AppError` strings), 17 and the second half of 10 (gate the - test-only storage surface), and the false `debug_assert` comment at - `sequencer-core/src/fee.rs:228`. -- **followup-1-submitter** (findings 1–3, ~250 lines), **followup-2-schema** - (findings 9, 11, 18, while the baseline-rewrite window is open), - **followup-3-measure** (the 500 ms objective and catch-up ACK p99), - **followup-4-harness** (the owed levers and the e2es they unlock, split by - lever), **followup-5/6/7-track3** (typed history foundation and - `GET /history-version`; finalized replay routes, gated on consumer decisions - 2 and 3; the `/ws/subscribe` cutover absorbing findings 6 and 7), - **followup-8-track6** (working-image `Application` API; breaks the same trait - this PR breaks). - -## Recommended split - -**In this PR before it leaves draft:** the CI fix (binary ANSI plus the exit -code assertion), the PR title, body, and breaking-change notes, the two doc -lines that go false on merge, the prose-versus-types honesty sweep (token -coverage claim, "non-clone", the witness wording, the three "journal" words, -the "acknowledge" mention, the README type name, the two-row black-box shape), -the five mechanical register items, gating `ensure_open_tip`, and deleting the -panicking progress constructor. Each touches files the branch already rewrites -and none changes behaviour a reviewer has not already seen. - -**A focused successor PR:** the remaining confirmed items — `TipAlreadyOpen`, -the notification-half narrowing with its doc restatements, the kind-filtered -key-file error, the tip postcondition refuse, the exit-code table with the two -process-level assertions — plus whichever unverified runtime items survive -their own re-verification (the lease-release supervisor's home, the -non-optional inclusion block, the in-scope recorder). These change behaviour -or exit-code classification and deserve their own adversarial pass and tests. - -**Later, in order:** the doc single-home passes; submitter pacing; schema -hardening before first deployment; the latency measurement; the harness -levers; Track 3; Track 6. diff --git a/docs/review/2026-09-09-application-lane-dex-review.md b/docs/review/2026-09-09-application-lane-dex-review.md deleted file mode 100644 index a4bfa345..00000000 --- a/docs/review/2026-09-09-application-lane-dex-review.md +++ /dev/null @@ -1,389 +0,0 @@ -# Application, inclusion lane, and DEX integration review - -Status: accepted and implemented locally. The investigation below records the -pre-change evidence. Current contracts live in -[application-contract.md](../protocol/application-contract.md) and the snapshot -docs. The reference C bridge is ported on a separate integration branch, not -cherry-picked into the main implementation. - -## Accepted follow-up decisions - -- Keep `Send`; remove unused `Clone + Sync`. Independent state forks use - checkpoint/restore, not a required `Clone`. A non-Clone, non-Sync runtime - fixture exercises actual preparation and launch. -- `progress()` returns engine-owned count/clock by value. Apply hooks advance - it; the shared boundary preflights overflow and verifies successful - transitions. Validation distinguishes rejection from fatal engine failure. -- Mutable `create_dump` permits backing-resource changes while preserving - logical state. File and directory prefixes remain opaque. Durable immutable - checkpoints, independent restores, and source-deletion independence are - required; public flush/clone/reopen machinery is deferred. -- Recovery state and canonical comparison bytes may differ. The wallet uses - the same binary SSZ file for both; the DEX design compares `M` with its - canonical drive. The current watchdog's drive extraction remains follow-up. -- Canonical inspection is a separate trait. Lane bookkeeping is simplified - without changing ordering, attempt limits, commit/ACK, or drain/promotion - atomicity. CORS and Lua executable parity remain a separate focused commit. - -The private DEX scheduler and native engine are still unavailable. Reference -bridge tests verify the proposed seam, not private engine conformance. DEX -conformance and the [Track 3 history API](../plans/2026-07-track3-feed-replay-design.md) -remain follow-ups. This review establishes reference integration coverage; -it does not establish that the Application surface is production-proven. - -Reviewed on 2026-09-09: - -- Local PR #28 work: `7d6238ab2ae1aadee6a858806747b042d87b896c`. -- DEX integration branch: `c31bf18413d8e9677ad9d663b9a045f44fafa4a3`, - compared from common ancestor `993d3310ac0da63380a62d6d3c93bff22d2317ff`. -- [C application bridge, PR #32](https://github.com/cartesi/sequencer/pull/32): - `0fa1755a882ce8aeeb6ba7877ba4ea7c479da9d1`, based on `01030fd7107e360d9a4d7e0e2852183eadb4b327`. -- Uncommitted lane annotations in the main checkout were read without alteration. - Its unresolved index entry was not resolved by this review. - -The actual DEX bridge and C++ scheduler have not been shared. PR #32 is evidence -of the intended integration approach, not proof of the private implementation. - -## Findings and proposed Application boundary - -### 1. Native progress ownership fits the supplied bridge - -The current trait calls progress scheduler-owned, stores it inside the app, -requires immutable and mutable references to it, and controls the latter with a -separate capability. Execution then checks that application hooks did not change -it and that the getter and mutator agree. See -[`Application`](../../sequencer-core/src/application/mod.rs). - -PR #32 describes a different, coherent ownership model: native execution advances -and persists count and clock, and two C functions read those values. The adapter -holds an opaque engine pointer. Mapping this directly onto the current trait -would conflict with the assertion that native execution must leave progress -unchanged. A Rust shadow value would introduce two representations of the same -fact and require synchronization at save/load boundaries. - -Recommend treating `Application` as the complete execution engine: - -- Read progress by value: `progress(&self) -> ApplicationProgress`. -- Let successful native execution update its own complete state, including - progress. The protocol still defines the transition. -- The shared Rust execution boundary preflights the checked count successor, - invokes execution, checks the exact expected count/clock after success, and - returns the pre-execution offset. -- Remove the mutable progress accessor and both capability types. Remove - progress-only validation-purity and post-error coherence checks; an execution - error already defines no successor and the instance must be discarded. -- Preserve `clock = max(previous_clock, input_block)`, count zero implying clock - zero, durable round trips, replay attribution checks, and deterministic - application behavior. - -PR #32 can implement the value read using its existing two scalar getters. No -C ABI change is required for that read, and the single-owner handle prevents -concurrent mutation between them. Rust implementations may share a small -progress-transition helper; the C++ implementation follows the same formula. - -This deliberately gives up capability-enforced routing for safe Rust callers. -Production call sites must use the shared boundary and conformance tests must -verify the native contract. The current capabilities never prove validation: -`execute_valid_user_op` accepts a publicly constructible `ValidUserOp` because -trusted replay needs to bypass admission. They are not an FFI isolation boundary. - -The earlier blanket rejection of `AppWithProgress` in the review register was -too broad. A wrapper shared by canonical execution, the lane, and recovery could -own progress correctly, with codecs preserving existing canonical bytes. The -invariant rejects an off-chain-only sidecar, not composition. Nevertheless, that -would require coordinated ownership and codec changes, while the native-owned -value interface fits PR #32 without inventing a second owner. Prefer the latter. - -### 2. Remove unused Clone and Sync requirements - -The entry chain in [`harness.rs`](../../sequencer/src/harness.rs), -[`run/mod.rs`](../../sequencer/src/commands/run/mod.rs), and -[`workers.rs`](../../sequencer/src/commands/run/workers.rs) requires -`Application + Clone + Sync`. Prepared runtime state holds no application value; -the lane constructs the application inside its blocking thread. - -This already has a concrete cost in PR #32: `EngineApp` supplies a panicking -`Clone` and an `unsafe Sync` justified by the current runtime never sharing it. -The C ABI forbids concurrent use of a handle. A public `Sync` implementation -licenses shared-reference calls from multiple threads, so current call-site -discipline is not a sound general justification for that implementation. - -Removing all five `Clone + Sync` bounds compiled across the complete workspace -and all targets in an isolated archive. Remove the bounds and those adapter -workarounds. PR #32 explicitly permits moving handles, so `Send` is supported by -this ABI; there is no need to redesign thread confinement for this integration. -Its necessity belongs to the async host, rather than canonical transition -semantics, if a future engine is thread-affine. - -### 3. Preserve all three validation outcomes - -Rust validation currently returns only success or `InvalidReason`. PR #32's C -validation function returns OK, INVALID, or INTERNAL. The adapter therefore has -to abort when validation fails internally. - -Recommend `Result`, where `ValidationOutcome` is -`Accept` or `Reject(InvalidReason)`. The shared protocol guard still checks -`max_fee >= current_fee` first. Expected rejection remains a nonmutating response; -internal failure propagates to the host's failure policy. - -Keep validation and execution separate for now. Catch-up currently replays -`ValidUserOp { sender, fee, data }`, without nonce or max-fee fields. Combining -the methods would require rebuilding original operations and checking their -admission result on replay. That is possible, but PR #32 already supports the -split, so it has no demonstrated benefit here. - -The bridge's rationale for aborting execution errors predates the local scheduler -fix: the reviewed scheduler propagates application errors instead of swallowing -them. The adapter can now report execution failures as `AppError`; process policy -belongs to the host. Exceptions must still never unwind across the C ABI. - -### 4. Fix the rejection contract before asking clients to implement it - -The current application contract recommends `ExecutionOutcome::Invalid` for -malformed application payloads, but application execution hooks cannot return -that type. The wallet consumes nonce and fee before decoding a method; malformed -methods and unsuccessful business operations return `Ok` with no outputs. -Malformed or unsupported direct inputs likewise execute as counted no-ops. - -The distinction to document and test is: - -| Outcome | Included? | Progress | Meaning | -|---|---|---|---| -| Admission rejection | No | Unchanged | Invalid nonce, insufficient fee balance, or max fee below frame fee | -| Included business failure/no-op | Yes | Advances once | Method fails under application rules; fee/nonce behavior remains the application's defined included semantics | -| Internal execution failure | No canonical successor | Instance discarded | A bug or unrecoverable execution failure, never an ordinary bad DEX order | - -Do not change fee/nonce or rejection semantics as part of this interface cleanup. -Also correct two smaller documentation errors: ingress does enforce the declared -payload bound, and the wallet no longer repeats the shared max-fee check. - -### 5. Keep checkpoint requirements, remove representation assumptions - -Retain the simple lifecycle: load state; create an immutable, crash-durable -checkpoint; locate its canonical file; dispose of obsolete state. A returned -checkpoint must be durable before SQLite references it. HTTP readers retain -their leases, including for directory-shaped dumps. - -PR #32 describes private live mutations over an immutable source image and an -explicit durable save. It does not require a caller-managed write-through -working image. The Track 6 draft's `open/flush/clone` lifecycle should not be -imposed on this integration. CoW and sparse writing can remain engine details. -Measure checkpoint tail latency before adding asynchronous staging. - -The sequencer-owned outer dump is a directory containing `info.toml`. Its -application prefix can already be treated opaquely by create/load/delete and -canonical-file lookup. Explicitly allowing that prefix to be either a file or a -directory would accommodate PR #32 without requiring a dummy subtree. Pin this -with a single-file lifecycle fixture if adopted. Preserve directory support. - -The load contract should explicitly state how an instance remains usable after -its source checkpoint is collected. PR #32 promises that property through its -private mapping; other implementations must provide equivalent independence. -Do not generalize that implementation into an assumption that every app consists -of one mapped file. - -Durable deletion is unnecessary for the sequencer's safety: SQLite references -are removed first, and orphan files after a crash are acceptable. Durable creation -remains necessary. Preserve meaningful load-error classification: PR #32 maps -every IO_ERROR to `ErrorKind::Other`, losing the missing-artifact distinction the -current host uses to refuse a broken referenced checkpoint. Carry the needed -typed distinction across the ABI rather than parsing diagnostic text. - -### 6. Put optional capabilities on their actual consumers - -`export_state` has no generic Rust consumers; keep human-readable debugging on -the concrete app. `canonical_snapshot_bytes` belongs to canonical inspection, -not every native engine adapter. The Rust scheduler's inspection method should -require an inspection capability where used. A separate C++ canonical scheduler -can provide inspection itself while the sequencer serves the canonical file. -Do not turn the current default runtime error into a globally mandatory bridge -method merely to make the trait uniform. - -Execution and durable checkpointing are distinct contracts with actual distinct -consumers. Separating those traits is reasonable if it makes the implementation -clearer, but no generalized capability or lifecycle framework is needed. - -## Inclusion lane - -No new supported canonical-order or acknowledgment correctness defect was found -in the reviewed lane, replay, snapshot, and storage paths. - -The useful simplifications are local: - -1. Collapse `ChunkOutcome`, the accepted-count return, and `FastTurnSummary` - into one bounded-turn result. Preserve the cap on attempted requests, so a - rejected flood cannot starve reconciliation; preserve commit-before-ACK. -2. Store `next_safe_input_index` rather than the previous complete - `last_drained_direct_range`, whose end is the only subsequently used value. - Advance it only after a successful commit. -3. Consider one storage frame-transition operation with an optional promotion - argument, replacing the duplicated promoting/nonpromoting entry points. - Drain attribution, progress mapping, frame creation, and promotion must - remain in one transaction. The observation accumulator remains useful. - -Answers to the in-tree annotations: - -- `frontier_min_interval` limits SQL observation frequency (default one second). - User-op chunks continue during that interval. The five-safe-block criterion - controls logical frame advancement and deposit visibility; it is a separate - policy. Removing it changes behavior while saving little code. -- The durable divergence check must precede the clock threshold, including when - no new frame is due. It detects a poisoned accepted-batch projection. -- The reintroduced `is_storage_invariant_contained` check belongs to the old - global terminal mechanism removed from the reviewed HEAD. It historically - checked a different signal, not the same database fact twice. -- The divergence check is polling-based diagnosis. SQL does not fence every - post-marker ordinary frame/user-op append; do not describe it as doing so. - -Whole-range reconciliation remains appropriate under the explicit capacity -assumption: fix one safe frontier, execute its direct prefix, and atomically -attribute it to the advanced frame before later user ops. Paging bounds payload -scratch memory, not the full receipt vector or turn duration. Reconciliation, -checkpoint creation, and GC can delay overlapping requests. Measure them with -the DEX engine before introducing preemption or resumable ordering state. - -## DEX branch parity - -The five feature-only commits do not establish a large missing runtime surface. - -| Capability | Local status | -|---|---| -| WS user-op nonce, frame safe block, batch nonce | Present; same wire fields | -| Direct-input input index, batch nonce, block timestamp, transaction hash | Present; same wire fields and encodings | -| WS catch-up close reason carrying live-start offset | Present | -| Avoid historical getLogs when the safe input count has not changed | Present | -| HTTP transaction request and acknowledgment | Same schema | -| Browser POST /tx | Works; CORS policy differs | -| Explicit Lua 5.4 executable selection | Not carried over everywhere | - -The CORS discrepancy is concrete. The feature branch allows any origin, POST, -and request headers, with a 3600-second preflight cache, on ingress only. Local -`http.rs` applies `CorsLayer::permissive()` to the merged ingress/egress router, -including internal reads, and configures no preflight max-age. The branch's -egress-isolation expectation fails locally. Restore the narrow scope without -waiting for a port split; carry its error-path and preflight contract tests. - -Reconcile Lua executable selection with the supported Nix/native environments; -blindly replacing every invocation can break an environment that exposes its -pinned Lua 5.4 as `lua` rather than `lua5.4`. - -These larger items are absent from both public API variants, not lost fork -features: - -- Recovery-aware public history: local storage has canonical count and - era/generation, but WS and snapshot headers still expose physical rowids. -- Full remote recovery-dump export: local dumps support multiple files, while - snapshot HTTP routes serve one canonical file. -- Public application-output delivery: current WS sends inputs and POST returns - the inclusion acknowledgment, not notices or vouchers. - -Keep the established history-protocol work separate. Add archive export or an -output stream only for a concrete consumer requirement. PR #32's C engine -binding is also separate integration work, not already provided by the CORS -branch. - -## Scheduler integration and approach - -Sharing the DEX scheduler between its canonical machine and cockroach recovery -would remove an important independent implementation. The Rust scheduler should -not be presumed more correct. However, compile-time selection alone does not -remove all possible disagreement: the live lane and SQLite acceptance/nonce -projection still encode protocol assumptions and must agree with that scheduler. - -The eventual scheduler interface should be separate from per-application input -execution and should cover the actual recovery needs: restore a checkpoint, -seed pending directs, process L1 inputs, drain at the recovery stop, and return -application state and next batch nonce. Determine that interface from their real -scheduler rather than encoding the Rust implementation's convenience methods -into a new requirement now. - -Future microbatch priority is a different decision. It can select an order among -pending, unacknowledged operations before validation/execution and persist that -chosen order; it need not create a protocol frame every 500 ms. Preserve direct -drain attribution and validate sequentially against the chosen order. Under the -current nonce contract, a cancel at nonce 11 cannot simply move before a new -order at nonce 10 from the same sender. A full 500 ms collection window also -spends the entire advertised acknowledgment budget before execution and commit. -These are design constraints for that later work, not reasons for a policy -framework today. - -Recommended sequence: - -1. Simplify Application ownership and error outcomes, remove unused bounds, - and port the reference C bridge alongside the Rust wallet. Preserve canonical - bytes and existing rejection behavior. -2. Polish the lane's local bookkeeping with its current ordering intact. -3. Close the narrow CORS/tooling parity differences in a focused integration - change. Keep public history cutover as its own API change. -4. Integrate the real scheduler once available, then consider measured ordering - policy requirements. - -The main process improvement is to use the native adapter as an acceptance test -for an interface change. One source engine exercised through Rust, the C ABI, -replay, dump/load, and canonical execution exposes more useful integration -mistakes than additional capabilities around two self-trusted fields. Include -admission rejection, included no-ops, exact progress, output order, snapshot -immutability, and error classification. Cross-language scheduler tests should -compare behavior against the protocol, not automatically bless either side. - -## Validation and limits - -All commands used the pinned environment via -`direnv exec /Users/gcdepaula/projects/cartesi-dev/sequencer` from the reviewed -checkout unless stated otherwise. - -- `cargo check --workspace --all-targets --locked --offline`: passed. -- `cargo test --offline --locked -p sequencer --lib ingress::inclusion_lane -- --nocapture`: - 50 passed. -- `cargo test --offline --locked -p sequencer-core -p app-core --lib`: - 107 core and 23 app tests passed. -- Isolated archive, removal of all five `Clone + Sync` bounds: - full workspace/all-target check passed. -- Isolated archive, two real-listener tests adapted from the feature branch's - CORS expectations: reproduced both differences (`/livez` returns allow-origin - `*`; `/tx` preflight has no max-age). These are expected repro failures, not - failures of the existing suite. - -The existing cheap-app 5,000-direct backlog test took about 47 ms in this run. -That is neither a DEX benchmark nor a concurrent ACK latency measurement. -No private DEX engine/scheduler, native bridge runtime, or canonical-machine -end-to-end execution was validated in this pass. - -## Implementation validation (2026-09-09) - -The local implementation passes `cargo check --workspace --all-targets`, -strict workspace/all-targets/all-features Clippy, and formatting checks. -`cargo test --workspace --exclude canonical-test -- --test-threads=1` passed -697 tests; one pre-existing doc example remains ignored. Wallet SSZ golden -bytes are unchanged. The watchdog Lua 5.4 suite passed 62/62. - -New coverage includes pure validation, native progress mismatch and overflow, -fatal validation propagation, no reads after a failed apply hook, independent -wallet and file/directory checkpoint restores, atomic frame/promotion rollback, -a Send-but-not-Clone-or-Sync runtime, ingress-only CORS on success and rejection, -and POST preflight policy. The CORS fixture initially queried an uninitialized -snapshot service and correctly triggered a terminal fault; it now seeds the -required finalized checkpoint. An unrelated lock-lifetime test failed once in -a parallel run and passed alone and in the final serial workspace suite. - -CORS and Lua invocation changes close the reviewed public branch's remaining -narrow parity gaps. This does not deliver the separately planned history API, -remote full-dump export, output stream, or private scheduler integration. - -Real-process follow-up rebuilt the devnet binaries at `5773b833` and passed -`restart_and_replay_test` with deposit, transfer, withdrawal, and restart replay. -`setup_recovery_round_trip_test` restored checkpoint `B=26, N=1`, accepted the -continuing nonce, passed its anchor/divergence checks, and finalized the resumed -snapshot at block 35. The test then failed during watchdog initialization: -`expected "archive_version" 7 (got 6)`. The installed in-process Lua Cartesi -binding expects the newer archive, while the repository image is pinned to -CM 0.20. The verified CM 0.20 CLI shim does not affect that Lua binding. This -E2E remains incomplete; no emulator pin or image was changed to make it pass. - -The separate reference bridge integration also exposed a concrete host need: -lazy genesis construction must be fallible, and a custom CLI must reuse the -library's command-task exit projection. Its integration branch makes the -factory return `Result` and exposes `run_command` for parsed -commands. Completed setup still avoids opening the original genesis. This -keeps file-load errors in the existing bootstrap error policy without adding -a second setup-admission check in the C host. diff --git a/docs/review/2026-09-16-track3-validation.md b/docs/review/2026-09-16-track3-validation.md index d44cdb78..56aeaa0e 100644 --- a/docs/review/2026-09-16-track3-validation.md +++ b/docs/review/2026-09-16-track3-validation.md @@ -5,13 +5,15 @@ Scope: validate application-history commit a complete cold replica, and a same-host latency comparison against `91e25780854bb641c63135751f951f9f7ee1e744`. +Retained for the [Track 3 integration gates](../plans/2026-07-track3-feed-replay-design.md): +this is the wallet baseline against which native-engine and deployment results +can be assessed. Replace or delete it when those decisions no longer use these +measurements. It describes the named revisions, not ongoing validation of HEAD. + ## Environment and canonical agreement -The shared development flake was switched to emulator 0.20.0 using its previously -recorded source, generated-files, and uarch hashes. The CLI, Lua module, and -native library all resolved to the same Nix package. Existing unrelated Foundry -edits were preserved; the shared flake lock was unchanged. This environment edit -lives outside the sequencer repository. +The CLI, Lua module, and native library used emulator 0.20.0 from the same Nix +package in the shared development environment outside this repository. Rust 1.95.0, Lua 5.4.7, and Foundry 1.5.1 were used. A fresh devnet canonical image was built from this checkout with the pinned cross image and kernel; @@ -34,14 +36,12 @@ All four selected canonical-machine gates passed: | `setup_recovery_round_trip_test` | Real `/finalized_snapshot` download, database wipe, `setup --recovery`, resumed execution, and independent CM comparison | 15.36 s | These are selected integration gates, not a claim that the entire E2E suite or -private DEX adapter was tested. The existing 693-test host-suite result belongs -to the implementation record in the register. +private DEX adapter was tested. ## Cold replica -Added `cold_replica_snapshot_backlog_live_recovery_test` and two small wallet -replay helpers. The scenario passed in 18.12 seconds, including its test-owned -120-second deadline. Its claims come from HTTP headers and consumed inputs, +`cold_replica_snapshot_backlog_live_recovery_test` passed in 18.12 seconds, +including its test-owned 120-second deadline. Its claims come from HTTP headers and consumed inputs, without querying storage for the consumer's history identity. The test restores a nonempty tar archive, deletes the downloaded source, and @@ -58,17 +58,6 @@ The expected replacement branch retains the accepted prefix and replays only its retained L1 directs. Optimistic transfers disappear, and a new transfer at the recovered nonce succeeds. -## Tooling fixes - -- `just doctor` preserves Lua's configured search paths, matching the production - watchdog. Its forced Linux-only environment variables hid the Nix Cartesi - module. The corrected doctor loads both lcurl and the new machine image. -- Benchmark CLI/recipe defaults use `max_fee=2000`. The former 1200 default was - below the self-contained frame fee of 1356, rejecting every request. The stale - `--from-offset` help example was also corrected. An initial benchmark attempt - with 1200 was discarded during warmup; both compared revisions use an explicit - 2000 limit. - ## Latency comparison Four release-build runs used an ABBA order: baseline, current, current, @@ -77,6 +66,8 @@ baseline. Each had 5 seconds of warmup and a 45-second measured window, explicit max fee of 2000, a 3-second request deadline, and a 5-second WS deadline. The host was an Apple M5 Max (18 logical CPUs, 36 GiB) running macOS 26.6.2. No builds, correctness tests, or injected network shaping ran during measurement. +An initial attempt with max fee 1200 (below the frame fee of 1356) was discarded +during warmup; all four compared runs used the explicit 2000 limit. Both exact revisions used their matching SDK/protocol and the same fresh machine image, Anvil fixture, and toolchain. Per-request latency excludes funding, diff --git a/docs/review/README.md b/docs/review/README.md new file mode 100644 index 00000000..93c8e985 --- /dev/null +++ b/docs/review/README.md @@ -0,0 +1,60 @@ +# Review notes and their lifecycle + +Reviews help us investigate code and hand work over. Their conclusions must +survive where future contributors need them; their working notes need only +survive while useful. Git is the archive. + +## During a review + +Create a note only when the investigation or handoff needs one. A small review +can live in the conversation, PR, or commit description. Commit a note when +sharing the ongoing reasoning is useful; committing it does not make it a +permanent document. + +Notes are mutable. Correct, consolidate, and remove superseded claims instead +of appending a conversation transcript. Record the reviewed revision, scope, +evidence, uncertainty, and next question. Check behavior in code and tests; +documentation and earlier verdicts are leads, not proof. Separate observed +defects from coverage gaps, accepted tradeoffs, and unverified hypotheses. + +## Closing or handing off + +Give each surviving conclusion one home: + +| Conclusion | Home | +|---|---| +| Current contract, invariant, or durable design reason | Its owning design document, source comment, or test | +| Unresolved defect or investigation | [Current register](register.md), unless already owned by an active plan | +| Coordinated implementation or integration work | The relevant active plan; link to it from the register if useful | +| Measurement or other evidence still used by a decision | A dated record with the consumer, exact revision, method, result, limits, and retirement condition | +| Completed discussion, superseded proposal, closed finding | Git history; remove it from the working tree | + +An unresolved entry states the impact or question, supporting source/test, +last-checked date and revision, and next action or revisit condition. A shared +verification stamp is sufficient for entries checked together. A missing test +needs a specific unverified behavior; an absent test name alone creates no +obligation. Proposed machinery needs an invariant and supported assumptions. + +Preserve the reason for a deliberate tradeoff at its design seam, together with +the assumptions that would change the decision. A previous rejection is not a +permanent ban on an alternative. Avoid a second register of settled/refuted +decisions, closed-item tombstones, or review-codename maps. + +Delete the finished note after checking that useful unresolved work and unique +evidence have a home, and update incoming links. Apply the same rule to +completed plans. Revisit retained notes when related code changes, at handoff, +or when they stop informing a decision; no calendar-driven archive is needed. +Cleanup does not imply the remaining code work is complete. + +## Recovering earlier reasoning + +The last version of a deleted note remains available without loading it into +every agent's baseline context: + +```sh +git log --all -- docs/review/FILE.md +git show REMOVAL_COMMIT^:docs/review/FILE.md +``` + +Treat that version as evidence about its reviewed revision. Recheck its +premises before carrying a conclusion into current work. diff --git a/docs/review/register.md b/docs/review/register.md index 8942aa8a..90c9713a 100644 --- a/docs/review/register.md +++ b/docs/review/register.md @@ -1,709 +1,123 @@ -# The Review Register - -The distilled outcome of every dated review ledger in this directory: what -is still **open**, what was **settled** (with its reasoning's current home), -and what was **refuted** and must not be re-proposed without new evidence. -Check the open section before touching related code; check the refuted -section before proposing a simplification or a new mechanism. Every -review's date, scope, and verdict is in the Review history table at the end -of this file; the two August 2026 ledgers and the 2026-09-03 stock-take stay -as separate files because they carry measurements and refutation evidence -recorded nowhere else. Process detail beyond that lives in git history. - -Statuses of findings 1–18 were verified against the tree on 2026-08-25. -Findings 19–31 and the 2026-09-03 refuted block come from the -[branch stock-take](2026-09-03-branch-stocktake.md), which records every -proposal that review raised, including the ones no jury examined. - -The 2026-09-07 maintainer-approved simplification changes two earlier -premises: terminal runtime faults abort immediately, and startup recovery -uses an ordered procedure. Earlier dated containment/token and phase-driver -entries below record their historical rationale; those mechanisms and their -test-only obligations are superseded by the current ADR and recovery design. - -## Findings - -Code findings, oldest first (file references are starting points, not exact -lines). Numbers are stable identifiers: a closed finding keeps its number and -is reduced to a one-line closure note, so citations in git history and the -remaining dated ledgers stay valid. - -1. **Submitter confirmation-timeout defeats pacing** — a watch timeout maps - to a successful `Submitted` tick, so the next tick immediately re-sends - the same payloads at the same nonces (usually "replacement underpriced") - before any sleep. `l1/submitter/poster.rs` + `worker.rs`; add a distinct - outcome that sleeps. -2. **A transient `SQLITE_BUSY` costs the submitter a full respawn** — 50 ms - reader `busy_timeout` plus every non-poster error ending the run. It now - classifies restartable rather than terminal, but the respawn+recovery - cost stands. Retry BUSY or use the writer-grade timeout for these reads. -3. **An undecodable own-sender payload stalls submission for the safe-lag** - — the poster hard-fails decode where both scheduler mirrors - skip-and-continue, so an operator's manual tx from the submitter EOA - wedges ticks until the block passes the safe head. Skip undecodable - own-sender payloads. -4. **Closed** (2026-09-03): the flusher's healthy retry pass logs at - `warn!`, not `error!`. -5. **Closed** (2026-09-03): the application-error 500 body is the fixed - "application internal error"; the reason stays on the lane error and the - log. -6. **WS session hygiene** — a mid-session transient read error tears down - without a close frame. Ahead-of-head admission is closed (2026-09-16): a - typed HTTP409 refuses it before upgrade. -7. **Closed** (2026-09-16): mandatory era/generation/application-count claims - reject resume across recovery; snapshot headers provide cold-bootstrap - coordinates. Current application suffix replacement is atomic with the - generation bump. See the [history contract](../protocol/application-history.md). -8. **Fee-determinism contract under-specified** — the LSB-first - floor-after-each-multiply order is implemented but not stated as contract - (`sequencer-core/src/fee.rs`). Load-bearing for the C++ scheduler port; - interacts with the deferred fee-LUT track. (The `fixed_mul` comment that - claimed a nonexistent `debug_assert` now states what the truncation - relies on: the `MAX_EXPONENT` bound upstream — closed 2026-09-03.) -9. **`trg_enforce_nonce_contiguity` NULL hole** — a dangling parent makes - the comparison NULL and the trigger silent; mitigated by `foreign_keys=ON` - on every writer connection, but the trigger itself is not NULL-safe. -10. **Closed** (2026-09-16): batch sealing asserts that the next frame retains - the durable Tip clock; complete L1 reconciliation owns clock advancement. -11. **Partially closed** (2026-09-16): `safe_accepted_batches.inclusion_block` - drives accepted snapshot selection and export. `first_frame_safe_block` - remains audit-only and may be removed in a separate cleanup. -12. **`direct_q` is unbounded in the shared scheduler** — an adversarial - deposit flood is bounded in time (force-drain) but not bytes; a - per-input cap or byte budget closes a (very expensive) guest-OOM vector. -13. **`MAX_BATCH_METADATA_BYTES` (71) understates real SSZ per-op overhead** - (~83+ with offsets) — byte budgeting undercounts ~15% for max-payload - ops (`sequencer-core/src/user_op.rs`). -14. **Wallet snapshot decode accepts unsorted entries** while encode sorts — - enforce strictly-ascending addresses (subsumes the duplicate check) or - drop the canonical-decode pretense (`app-core/src/wallet_snapshot.rs`). -15. **Reader and submitter re-open `Storage` per tick** — a held connection - per worker drops per-tick overhead. Low priority. -16. **`should_retry_with_partition` substring-matches the Debug format** — - consciously accepted and regression-pinned against alloy's format; - revisit with structured JSON-RPC codes. -17. **Closed** (2026-09-03): `latest_batch_index`, - `ordered_l2_txs_for_batch`, and `promote_finalized` are `#[cfg(test)]` - (gated rather than deleted — the first two have callers across three test - modules). `safe_input_end_exclusive` has a live reader-path caller and - stays. -18. **`frames` lacks the immutability triggers `batches` got** — `fee` and - `safe_block` are documented immutable but convention-protected only. -19. **Closed** (2026-09-04): the key-file read returns - `BootstrapError::KeyFile { path, source }`, classified by kind - (`config::key_file_io_is_terminal`: missing, unreadable, not a file, or - not text → 30; environmental I/O → 1); the message names the path, never - the contents; both halves pinned. The sibling `create_dir_all` at each - command's start keeps `CommandError::Io` → 1: it creates rather than - reads, so a missing path is not an operator mistake there; a read-only or - wrong-type parent is, and is not yet separated. -20. **Closed** (2026-09-04): `ensure_open_tip_for_recovery` splits its - disjunction — a non-`Safe` danger raises `StaleDecision`, an already-open - Tip raises the payload-free `TipAlreadyOpen`, paired with a retry reason - so the operator sees it; the guard test and the polarity pin assert it. -21. **Closed** (2026-09-03): `Storage::ensure_open_tip` is - `#[cfg(test)] pub(crate)`; the two intra-doc links and the snapshot - lifecycle doc then named the reducer's guarded `EnsureOpenTip` phase. - That phase-driver description was superseded by ordered recovery on - 2026-09-07; the guarded storage operation remains. -22. **Closed** (2026-09-04): `DangerDetector`, `InputReader`, and the - fee-oracle worker (narrowed with them: same single use, same - construction-required lock) take `ShutdownSignal`; `launch` passes - `shutdown.signal()` for the three and the scope to the lane, server, and - submitter. Data-directory ownership is each worker's own `ProcessLock`, - so the watchdog's weak witness is unaffected; the doc comments now say - workers that externalize or contain take a scope. - Historical containment/watchdog rationale: superseded by the - [2026-09-07 immediate-abort model](../plans/2026-08-authority-boundary-adr.md#1-runtimescope-structured-process-ownership). -23. **Closed** (2026-09-04): the guarded `EnsureOpenTip` phase re-reads - `has_valid_open_batch` inside its own transaction and returns - `TipMissingAfterOpen` (classified `refuse`, exit 30) rather than commit - without a Tip; `drive_recovery`'s doc records the ≤5-phase bound and - `admission.tla`'s comment names the enforced postcondition. - The phase driver and its bound were superseded by ordered recovery on - 2026-09-07; the storage postcondition remains. -24. **Closed** (2026-09-04): the in-scope recorder is deleted, so a - contained run writes exactly one `terminal_faults` row — the command - bracket's, at settlement. Containment writes nothing durable. The - accepted loss, stated in the runbook: any death before settlement (an - abort at the terminal abort deadline, a controller panic, SIGKILL) leaves - only the process logs, and the single write has no second attempt. The - row is telemetry; restart policy is the exit code. - Historical settlement behavior: superseded on 2026-09-07. Terminal - runtime faults now abort immediately without a drain or terminal row; - terminal command errors returned through the bracket still get a - best-effort row. -25. **Closed** (2026-09-03): the token's doc, the ADR, and the settled - entry state its true scope (three compile-forced primitives; the rest - hand-placed); "non-clone" became "boot-local"; the witness comment names - every holder; the "journal", "acknowledge", and `DangerDetectorExit` - remnants are gone. - Historical token documentation: the token was removed by the - 2026-09-07 immediate-abort model. -26. **Closed** (2026-09-04): `acquire_finalized_lease` returns its own - `FinalizedLease { inclusion_block: u64, dump: LeasedDump }`, so the - `NOT NULL` column is no longer an `Option` and the impossible-`None` - containment branch in `finalized_state` is gone. Corrupt-row containment - is unchanged (the persistent-storage classifier and the decode panic, as - `corrupt_finalized_snapshot_trips_terminal_storage_fault` pins). - The lease type remains; the historical containment response was - superseded by immediate terminal abort on 2026-09-07. -27. **Closed** (2026-09-04): the two-verdict `RecoveryFailure::Provider` - is split into `ProviderUnreachable` (retry) and `SignerMisconfig` - (refuse), constructed by a pure `classify_signer_provider` pinned on all - three arms; `classify_input_reader` documents its phase-dependent - polarity and pins `Bootstrap`/`Join` refused at startup, non-terminal - live. The setup-versus-run asymmetry it exposed is finding 32. -28. **Setup admission is implemented twice with different semantics** — - `preflight_lifecycle_command` (used by `run`/`flush`) and - `admit_setup_lifecycle` (`commands/setup/mod.rs`) each check the same two - facts; an already-complete plain setup is a no-op in one and - `NotAdmissible` in the other, and `load_setup_identity` re-checks - completion with a third error type. Unverified. -29. **`http.rs` hosts a runtime component** — the ~175-line supervised - lease-release queue with two containment sites belongs in the egress - snapshot module that owns leases; its queue is unbounded and capped only by - concurrent snapshot requests. Unverified; weight, not safety. -30. **Harness block-time notions are unreconciled** — four constants with an - unasserted relationship; `advance_live_frame_until_covers` can drive L1 - ~20× ahead of the process clock; `aging_open_tip_runtime_danger_zone_exit_test` - stages an impossible chain and greps the rendered log (the CI red). One - `ChainClock` authority plus a post-mining drift check; assert exit 10 - instead of the log. The log grep is verified first-hand; the rest is - unverified. -31. **`log_gas_price_updated_at_ms` is production-write-only** — written on - every refresh, read only by tests, yet the threat model cites it as the - telemetry that justifies having no expiry gate. Surface it (health field - or the transient-refresh warn) or drop it and fix the sentence. - Unverified. -32. **A deterministic L1 misconfiguration exits 1 under `setup` but 30 - under `run`** — `setup` wraps every non-`Provider` input-reader failure - — from `InputReader::new` (`setup/mod.rs:107-127`) and from both - `sync_to_current_safe_head` calls (`setup/mod.rs:260-267,535-542`) — as - a worker exit, whose `InputReaderError::is_terminal_invariant` treats - `Bootstrap` as the live-worker case (non-terminal), so it projects to 1. - `run` meets the bad-RPC-URL half through `classify_input_reader` and - refuses at 30; the discovery-time half (wrong contract, pre-v3 InputBox) - has no `run` counterpart, because `run` builds its reader with - `from_parts` and never re-runs discovery. `setup` is an - operator-run one-shot, so the harm is a wrong hint, not a restart loop. - Candidate fixes: classify `setup`'s bootstrap failure through the same - phase table as `run`, or give `setup` its own terminal variant for L1 - misconfiguration. Surfaced by the 2026-09-04 refuters; not yet decided. -33. **Closed** (2026-09-07): ordinary shutdown no longer cancels the reader's - in-flight blocking append. The reader joins it before returning, so the - final clean-exit check sees committed divergence and append failures - reach the supervisor. The actual worker-loop regression fails when - cancellation around the append is restored. - -Open maintainer decisions: - -- **What the 500 ms acknowledgement contract is *for*** — it is 8× above the - worst measured value and shapes nothing today; either it encodes - catch-up-overlap headroom (then measure that) or restate the objective. -- **Catch-up ACK-latency measurement** owed to the benchmark harness: ACK - p99 *during* an epoch-sized catch-up reconciliation turn (the in-crate - seed test digests 5,000 directs over a 7,200-block jump in one turn). -- **Track 6 with Bart**: the hardlink-suitability dispute and the - changed-era bootstrap contract (see the tracks doc). -- **C engine follow-up boundary** (2026-09-11): agree output storage and - checkpoint layout with Bart before replacing the drain protocol with output - arrays or the path callback with a static suffix. ABI version negotiation, - checked-in bindings, and linker policy also need concrete integration - requirements. Keep these separate from the port and the agreed contract - reductions; the private DEX engine and scheduler remain unavailable for - conformance testing. - -## Owed tests - -Statuses swept 2026-08-22 and updated through 2026-09-04. - -- **Arm-ordering discriminating test**: both `ClosedBatchInDanger` and - `TipInDanger` genuinely in danger; assert Closed wins (today pinned only - incidentally by an equally-aged fixture). -- **Fail-loud halves**: no test references `CatchUpError::NoSnapshot` or - `InclusionLaneError::NoOpenTip`. -- **`EstimatedBatchInDanger` e2e** (recipe: mine ~800 blocks, faketime - +30 min without mining, respawn → refusal with zero invalidations). -- **Process-level divergence scenario** via `respawn_until_stable` (storage - and reducer coverage exists; the end-to-end freeze/refuse loop does not). -- **Per-variant exit-code e2e assertions for classes 20/40/1** (those - failure-path e2es still assert only `!success()`). Landed 2026-09-04: 10 - in the aging-tip scenario, 0 through `stop_expecting_clean_exit` at the - healthy stop of `recovery_after_stale_batches`, and 30 in-crate through - the real command bracket (`harness.rs`, `run` on a never-set-up data - directory); the five verdicts are pinned to their integers in - `commands/error.rs`, so renumbering `EXIT_TERMINAL` fails the suite. -- **Closed** (2026-09-07): the phase→progress mapping and scripted driver - were removed. Procedure tests now exercise real SQLite inspections and - mutations, replacing only external Sync/Flush operations. The - `classify_input_reader` polarity pins remain. -- **Full-tear cascade on a recovered (anchor = `N'`) tree** re-rooting at - `N'` (anchor unit mechanics are covered; this end-to-end shape is not). -- **Uniswap-mode fee oracle end-to-end**: every fixture and e2e pins fixed - mode, so no Uniswap-mode sequencer boots in tests. Setup validation, - RPC-free runtime source construction, transient quote retention, and - terminal misconfiguration are source-boundary-pinned in-crate as of - 2026-08-25; a real E2E still needs a mock pool — decide whether that extra - harness is worth its weight. -- **True same-block direct-input ordering end-to-end**: the renamed - `multi_deposit_reconciliation_test` covers multiple accumulated directs, but - default Anvil automining puts its portal deposits in distinct blocks. A real - same-block test needs queued portal sends, one explicit mine, equal receipt - block assertions, and WS order/block attribution. -- **Verify-then-write-or-strike** (status uncertain on 2026-08-22): the - encoded-wire-frame stamp at an advanced safe head; the wallet - insufficient-balance silent no-op and replay-determinism pins; the - young-never-submitted-batch cascade-policy pin; the `recover_aging_tip` - torn/no-Tip entry; the cascade-with-backward-clock pin. -- **Closed** (2026-09-16): I7's colliding-artifact test asserts that failed - snapshot registration rolls back the seal, successor Tip, and cached head. - Snapshot endpoint tests cover referenced artifact deletion as a terminal fault. -- **Harness levers to build with their tests**: pending-tx capture + - re-inject (`txpool_content`/raw-tx before `drop_all_pending_txs`, then - `eth_sendRawTransaction`) → unlocks the zombie e2e, the headline - adversarial scenario; snapshot lifecycle observability (DB readers for - the snapshot tables + dump-dir inspection) → the take/promote/GC/lease - e2e, plus finally *asserting* warm-resume-from-dump (every restart test - exercises it, none asserts it — a silent fall-back to genesis replay - would pass everything, just slower); a bare second Anvil - (`--chain-id `) + mid-run endpoint override → the wrong-chain - e2e; kill-at-log-marker → the flush-completion/cascade-commit crash - window; SQLITE_BUSY injection → the submitter/WS BUSY items. - **Recorded do-not-build**: a split-view/response-rewriting L7 proxy - (unit-level provider mocks instead; e2e validates only the passing - path) and fsync/power-loss WAL-rewind injection (out of scope — - state-construction variants cover the detectable halves). -- **External-engine conformance** (2026-09-11): a reusable runner needs - application-supplied genesis and meaningful accepted/rejected inputs; compare - canonical comparison files rather than byte-identical recovery dumps. - Native repeatability alone does not establish native/canonical-machine - equivalence. Publish fee-conversion data and vectors for the C++ integration - (finding 8), and add targeted devnet E2E coverage through the C host in CI. - -## Settled decisions - -Each entry: the decision, its reason, and where the reasoning now lives. - -- **Application-only current history** (2026-09-16): retain every raw L1 input - and original batch/frame/user-op record, but replace the invalidated flattened - application suffix. The recovered prefix is opaque. Mandatory offsets and - versioned claims replace the mixed replay log and sparse mapping. Acceptance - facts select immutable per-batch snapshots without promotion or restamping; - per-batch cadence and end-of-block watchdog comparison remain. The complete - model lives in [application history](../protocol/application-history.md), I5–I11, - I18/I20, and the snapshot lifecycle. - -- **No architectural restructure** (2026-06-10): one file per writer role, - `*_in(tx)` free functions composing into larger transactions, - storage-owns-SQLite / lane-owns-filesystem — the layout is sound and is - defended, not redesigned → AGENTS.md "Sequencer module layout", the - storage module docs (`storage/recovery.rs`), `docs/snapshots/lifecycle.md`, - and the do-not-simplify list in `docs/invariants.md`. -- **Write-before-broadcast watermark** (2026-06): the flush's completion - anchor is durable, not the local pool's memory → I14. -- **Content-identity check, gated on full acceptance** (2026-06): accepted - landings compare by content hash; content-equal copies are effect-equal, - so no batch identifier is needed; detection freezes the frontier - atomically with the detecting sync → I9, I15. -- **`synchronous=FULL`** (2026-06): externalization rides on commits, so - every commit fsyncs; noise-level cost on NVMe → `storage/open.rs` doc. -- **Cockroach recovery's flush is best-effort by construction** (2026-06): - the wiped DB destroys the watermark, so the flush resolves only what the - provider remembers, and plain `setup`'s detection gate shares the same - false negative. Accepted because the content-identity check turns the - residual zombie from silent divergence into a detected freeze (repair: - wipe and re-run). Recorded option if ever needed: an operator-supplied - flush floor from the old DB's watermark — fail-safe under corruption - (too high wastes no-ops; too low degrades to exactly best-effort) → - `cockroach.md` step 2. -- **Exit-code contract** (2026-06, panics-terminal amendment 2026-07): the - orchestrator must not parse logs; 10/20/30/40 by restart productivity → - `commands/error.rs` + the operator runbook. -- **Fail-loud check policy replaces "no defense-in-depth"** (2026-06): the - line is loud-vs-silent, not self-doubt; assertions must check real - invariants (the wall-clock CHECK cautionary tale) → the invariants check - policy. -- **Scoped pending clear** (2026-06): delete only pending rows at/above the - cascade pivot, in the cascade's transaction → I5. -- **Batch-tree anchor, not a sealed sentinel** (2026-06-25): the parentless - root carries the anchor nonce, exact-matched by the contiguity trigger → - I16. -- **`N` is trusted; no recovery-time verifier** (2026-06-26): a - sequencer-produced finalized dump cannot carry a wrong `N` by - construction; only wrong-low is caught at `run` → `cockroach.md` data - dictionary. -- **Anchor-aware frontier; recovery defers population** (2026-06-26): - below-anchor landings are trusted collapsed history → I15. -- **Recovery drain caps at `C`** (2026-06): `(C, H1]` deposits stay - undrained so `run` leads them exactly once → `cockroach.md` steps 3/6. -- **Don't resurrect TEST_PLAN.md** (2026-06): the scenario matrix rotted - once; owed tests live here as a dated, finite list. -- **The authority boundary** (2026-08, re-evaluated 2026-09-07): four - mechanisms — process ownership and terminal abort, fact-derived admission, - ordered startup recovery, SQLite-centered runtime with the two-regime lane → the - [ADR](../plans/2026-08-authority-boundary-adr.md). -- **Storage decode policy** (2026-07): fail-loud for contract-impossible - values; the named `saturating_query_bound` only where clamping preserves - the predicate → `storage/convert.rs` + the check policy. -- **The calibration rule** (2026-08-18): the complexity budget belongs to - concurrency, mutual exclusion, durability, and hostile-L1 robustness → - AGENTS.md design principles. -- **Terminal runtime abort** (2026-09-07, supersedes the 2026-08-18 - `Authorized` token): a diagnosed terminal fault logs and aborts, so there - is no terminal runtime to drain or gate. Ordinary shutdown signals, - concurrent joins, and process-lock ownership remain. Dedicated-process - hosting and prompt diagnostic logging are the supported assumptions; - orderly terminal requests and settlement are explicitly given up → ADR - mechanism 1. -- **Snapshot leases remain** (2026-09-07): application dumps may contain a - directory of state files. GC racing a download is an ordinary supported - operation; relying on open-file unlink behavior is not the general dump - lifetime contract → snapshot lifecycle, leases. -- **C bridge contract reductions** (2026-09-11): the linked engine reports its - stable payload bound at runtime, including a valid zero bound; C progress is - one count/clock record. This removes duplicated build configuration and - presents the protocol pair together without introducing another state owner. - Checkpoint artifacts are self-contained beneath their supplied prefix and - require only ordinary filesystem deletion, so the sequencer recursively - removes its enclosing dump directory without an application deletion hook. - Reader leases, SQLite-first deletion, and restored-engine independence remain - required → [C binding guide](../protocol/c-application-binding.md) and - [Application checkpoint contract](../protocol/application-contract.md#6-checkpoint-lifecycle). -- **C host failure boundaries** (2026-09-11): snapshot-path callback panics - enter the host's terminal-abort boundary; a failed HTTP task alone must not - leave sequencing active after an application invariant violation. Missing or - malformed genesis dumps and absent required options are terminal bootstrap - failures; operational I/O remains retryable. Genesis stays lazy: completed setup may - succeed after its original source has been deleted. A pre-command mandatory - state-file check would violate that lifecycle contract → snapshot handlers, - command error classification, and genesis harness tests. -- **Execution-offset continuity has one enforcement point** (2026-09-07): - the SQLite trigger rejects a noncanonical offset inside the application-input - transaction. The duplicate Rust loop was removed; rollback, invalidation, - and offset-reuse tests remain → I20. -- **Module homing** (2026-08-19): command brackets in `commands/` (with - config + the `CommandError` taxonomy), the capability substrate alone in - `runtime/`, `L1Config` in `l1/`; a full merge was refused because the - substrate is consumed crate-wide. `http.rs` stays whole until the - ingress/egress listener split forces it apart. -- **Lifecycle: facts govern; the black box records** (L1 2026-08-18 → - L2 2026-08-19 → L3 2026-08-22): admission is three facts; no state - machine, no acknowledgement (it carried no machine-consumed decision); - telemetry writes are verdict-neutral; terminal faults refuse at - re-detection with the residual recorded in the threat model → the ADR, - the invariants check policy, and the two dated ledgers. -- **Misconfig-poison taxonomy** (opened 2026-08-18, closed by L3): there is - no poison; misconfig is terminal by exit code only, and a fixed config - boots cleanly. -- **Submitter-key redaction** (2026-08-24): the key enters the process as - `SubmitterKey` at the clap edge — `Debug` redacts, no `Display` exists, - and the raw hex is reachable only through `expose_secret`, so every - consumer of the secret is greppable. The key's public identity is the - pinned `batch_submitter_address` beside it. Closes the former - Debug-derive open finding → `l1/mod.rs`. Deferred separately: the startup - log prints the full RPC URL, which the help-leak test treats as - token-bearing. -- **One home per mechanism** (2026-09-05, updated 2026-09-07): every mechanism has one canonical - statement, and every other site is a pointer or an explicitly scoped - partial. The homes: the authority-boundary ADR for `RuntimeScope` and - terminal abort (mechanism 1), fact-derived admission and the black - box (2), and the SQLite-centered runtime and the two-regime lane (4); - `docs/recovery/README.md` for the procedure; `docs/invariants.md` for the - cross-module invariants, the check policy, the writer roles, and the - divergence freeze (I15) with the check's completeness scope (I9); - `docs/protocol/scheduler-semantics.md` for the frame clock; - `docs/protocol/application-contract.md` §5 for the digestibility - assumption; `README.md` for the API contract and the storage model; the - schema for the write-once batch lifecycle; `commands/error.rs` for the - exit-code contract, with the operator runbook and README carrying the - operator- and user-facing lists; `runtime/shutdown.rs` and the runbook for - terminal stop policy; this register for refuted proposals and the review - history. Code-side exceptions, where the argument is falsifiable - only at the code: the stack-local flush observation, the admission - linearization (`admit_runtime`), and the one-transaction inspection - (`RecoveryInspection`) → - AGENTS.md "Documentation Practice". - -## Refuted — do not re-propose without new evidence - -From the 2026-06 reviews: - -- **`scheduler_accepts` omitting the two structural rejections is a bug** — - deliberate self-trust; the simulator runs only over our own well-formed - batches; the worst case is covered by the content-identity check - (documented in `scheduler-semantics.md`, duality-test-pinned). -- **Sealed `N'-1` sentinel batch** for recovery rooting — a valid closed - sentinel is a legal cascade pivot; nothing stops a runtime cascade from - invalidating it, after which recovery ABORTs. Its safety rested on an - unenforced assumption. -- **Recovery-time `N` cross-check** — circular: every cheap recomputation - seeds from the `N` it would check; the only independent check is a - from-genesis L1 replay, deliberately not built. -- **`FoldInputSource` abstraction** — a wrapper over a single call site; - revisit only if a second fold-input source appears. -- Wire-fee-exponent panics, stalled-WS DoS, operator `WalletConfig` dead on - warm start, snapshot bytes history-dependent — each verified as - deliberate/out-of-scope (see the threat model's scoping). - -From the ADR's rejected list (opened by the 2026-08-01/02 re-evaluation); -the 2026-08-18 premise challenge found these alternatives left no residue in -code: - -- **`RunEpoch`** (a globally threaded internal fencing epoch) — the OS lock - plus structured task lifetime plus fresh per-scope channels already make - an old sender unable to reach a new receiver, and there is no in-process - hot restart to fence against; persisted rows cannot distinguish a live - owner from a stale one, a kernel-held lock can. Revisit only if in-process - restart or multiple admitted runtimes under one lock are introduced. - Evidence: `runtime/process_lock.rs` (module doc); no epoch type exists in - `sequencer/src`. -- **`EffectGate` / `LiveKernel`** (a universal effect mutex or actor, with a - reader mailbox) — would duplicate the role-local linearization points the - system already needs and force the reader and the latency-critical lane - through a new in-memory authority protocol, adding a second state machine - without making the narrow content-identity check a complete divergence - oracle; SQLite stays the durable coordination plane. The `Authorized` - token is not this — see - [ADR mechanism 1](../plans/2026-08-authority-boundary-adr.md#1-runtimescope-structured-process-ownership). - Evidence: - `runtime/shutdown.rs` (`Authorized`); [I9](../invariants.md)'s - completeness boundary. -- **A generic command controller** over setup/rebuild/run/maintenance — - their facts are unrelated; combining them enlarges the cross-product state - machine without closing an enforcement hole, and a flush has no admission - state to restore or erase. Evidence: the per-command controllers are - separate — `recovery/mod.rs` (`drive_recovery`), `commands/setup/mod.rs` - (`admit_setup_lifecycle`), `commands/flush.rs` (the flush body); the one - shared piece is a *fact* gate, `commands/mod.rs`'s - `preflight_lifecycle_command` (used by `run` and `flush`; `setup` reads - its own two facts inline), which checks admission facts and reduces - nothing. -- **A per-chunk divergence query, provider call, or reader mailbox** on the - hot path (formerly proposed per user-op) — the content-identity check is - complete only for at/above-anchor accepted-batch content identity - ([I9](../invariants.md)), so a query paid on every user-op chunk would buy - no complete safety boundary. Cost is not the argument: the `POST /tx` - round trip that carries a chunk is about 13 ms at concurrency 1 - (concurrency-1 HTTP ACK p50 13.231 ms against submit-to-matching-WS-event - p50 25.313 ms in the same harness session — the ADR's - [performance posture](../plans/2026-08-authority-boundary-adr.md#performance-posture) - carries the surviving figures; the maintainer's earlier informal "roughly - 14 ms" localhost round-trip observation carried no metric qualifier), so - the query would be cheap and still incomplete; a provider call on the same - path would put L1 liveness inside the acknowledgement path. Evidence: - `ingress/inclusion_lane/mod.rs` (the bounded chunk and the time-gated - frontier read); I15's runtime reaction. -- **A durable recovery-phase ledger** — the flush and post-flush-sync - witnesses are boot-local by design; persisting them would re-create a - state machine whose only effect is skipping an idempotent flush, and would - let a restarted attempt trust a half-remembered phase. Evidence: - `recovery/mod.rs` (`RecoveryProgress` is memory-only and `drive_recovery` - its only writer; the pin - `reconstructed_controller_cannot_reuse_a_post_flush_sync_witness`); - `admission.tla` (no durable per-attempt record gates anything). -- **A marker-file containment protocol** (2026-08-01; hardened by that - review, then deleted wholesale — git history) — a filesystem side-channel - for containment state, superseded by the in-process containment bit and - the settlement-written black-box row for containment state (ADR - mechanism 1) and by fact re-detection at boot for the durable verdict - (ADR mechanism 2); the kernel process lock guarded it and outlived it. Do - not reintroduce one: SQLite is the durable coordination plane and the - process lock is the exclusivity primitive. - -From the 2026-08-18 adversarial pass: - -- **Merging `Workers::finish`'s two drain modes by re-awaiting the primary** - — the winning select arm consumed the handle's completion (the select - borrows `&mut self.server` etc., so the handle is still in the cleanup - set); re-polling it panics ("JoinHandle polled after completion"), - unwinding through `ShutdownOnDrop` into containment — a benign stop - becomes a poisoned data directory. Scope-narrowed 2026-08-23: the 2026-08 - source read "as sketched" / "the naive merge"; distillation dropped the - qualifier. A merge that removes the primary from the cleanup set before - draining — keeping the `expect` that it was present — is settled, not - refuted (landed as `finish`'s one-loop/two-phase shape). -- **Removing the post-commit accessor-coherence assertion** — it is the - only guard in the two contexts with no database backstop (canonical - RISC-V fold, `fold_replay`). -- **"The three-variant frame-drain writer family is bloat"** — backwards: - the raw physical writers are `#[cfg(test)]`-demoted; production has one - way to write a frame. -- **`FuturesUnordered` for cleanup polling** — not worth promoting a - dev-only dependency tree to delete one small hand-written future. - -From the 2026-08-23 run-glue simplification pass: - -- **`Workers` as a homogeneous component list** (a `Vec<(WorkerId, - ComponentShutdown)>` built at launch, one select arm racing the list) — - it makes the "no `.await` between `Poll::Ready` and `swap_remove`" - property `select!`-load-bearing and untested (violation loses a worker - exit *and* panics on re-poll); converts the asserted primary-in-set - precondition into an unchecked cross-function assumption whose failure is - the benign-stop-to-exit-30 outcome; deletes the composed detector - select-mapping tests; and makes a zero-component race representable. The - real hole it targeted (select arms were the one per-worker site not - compile-forced) is closed by the exhaustive `let Self { .. }` destructure - in `select_first_exit` instead. -- **`UniswapConfig::pinned(&identity)`** — the exhaustive - `FeeOracleIdentity` match in the run bracket IS the launch decision for - the optional oracle worker; moving it behind an `Option` - constructor makes a future identity variant compile while silently - launching no worker, and the chain-id-pairing guarantee it claimed is - already structural now that the identity travels whole inside `L1Config`. - -From the L3 review (2026-08-22): - -- **A durable boot gate on terminal verdicts** (a gate on a *verdict*, i.e. - a non-fact) — it needs an acknowledgement to exit, and the acknowledgement - carries no information the fact-derived reducer doesn't re-derive. A - verdict-neutral startup *read* of the black box is not covered by this - entry, and is now exercised: `run` logs the latest row once at startup and - branches on nothing (`warn_on_previous_terminal_fault`, 2026-09-04). -- **A boot-time full-integrity sweep** — expensive machinery that still - cannot catch semantic violations outside its read set; the residual - window is recorded and bounded instead. -- **A boot assert on `finalized_snapshot.inclusion_block`** — the column is - `NOT NULL` at the engine; the claimed gap does not exist. (The runtime - `Option` branch on the same column was finding 26, closed 2026-09-04.) - -From the 2026-08-25 fee-oracle lifecycle pass (143a290; a single-pass -decision, not an adversarial review): - -- **Fee-price age as a runtime lifecycle gate** — setup owns the required - live quote; run starts from the persisted price and refreshes it best-effort. - Shared-endpoint staleness is already detected from safe-head progress, while - a pool-only outage is an explicitly accepted economic residual. Reintroduce - an expiry gate only with an independently derived economic bound and action, - not by borrowing the L1 liveness threshold. The telemetry this entry leans - on is production-write-only (finding 31). - -From the 2026-09-03 branch stock-take (three refuters per proposal; full -reasoning in the [ledger](2026-09-03-branch-stocktake.md)): - -- **`Workers` on a `JoinSet` fed by the shutdown waiters** — `from_select` - and `from_shutdown` deliberately read a worker's clean `Ok(())` two ways - (live: stopped unexpectedly; drain: graceful); one set collapses them and a - silently dead lane becomes exit 0. Evidence: `commands/error.rs:606-627`, - `commands/run/workers.rs:339-390`. The `swap_remove` hazard is real; a - cleanup-only set built inside `finish`, with the live race untouched, was - not examined. -- **Collapsing `PreparedRuntime` into an `async fn boot`** — - `fn launch(self, RuntimeAdmission) -> Workers` is non-async and non-`Result`, - so `?` and `.await` after admission are compile errors today; `boot` makes - them legal and the witness degenerates. Evidence: `workers.rs:288`, - `recovery/mod.rs:477-483`; 02a2b34 stopped here deliberately. -- **Making the `Authorized` token "uniform" by demoting the `/tx` 200 gate - to a bool** — the 200 body is the acknowledgement leaving the process; - `LeasedDumpBody` does not exist and `finalized_inclusion_block` has no - streaming primitive. Evidence: `ingress/api.rs:84-92`, - `egress/api/snapshot.rs:102-121,209-214`. The doc tightening survives (finding - 25). -- **Deleting the `#[from]` impls so a refusal reason cannot be typed into a - retry** — the enums are public with public variants; the longer spelling - still compiles and is the dominant idiom at all 15 sites. Evidence: - `recovery/mod.rs:45-49,122-128`. -- **One `is_terminal()` on `VerifiedSignerProviderError` read by both - tables** — the `From` impl performs no classification; the verdict is taken - later over `BootstrapError`, whose variants have four other producers. - Evidence: `commands/error.rs:688-700,220-273`; `l1/reader.rs:96-104` - already documents the phase pair. -- **Flattening `RecoveryError` into one enum with `is_retryable()`** — - refuted 2026-09-03 because `RecoveryFailure::Provider(String)` carried two - verdicts, so no total function over the value existed. That premise was - retired 2026-09-04 when finding 27 split the variant; every - `RecoveryFailure` now determines its verdict from its value. The entry - keeps its place on its remaining ground: the `Retry`/`Refuse` wrapper is - the reducer's verdict at birth, not a predicate a consumer recomputes, and - dropping its `Box` grows the `CommandError` footprint managed against - clippy's `result_large_err` (`commands/error.rs:537`). Re-propose only - against those. Evidence: `recovery/mod.rs:122-128,545-583`. -- **Moving the phase→progress mapping into `drive_recovery`** — `(Flush, - Done)` has no target without the observed block; this is - `PhaseCompletion`/`transition_after_phase`, deleted by ed41f9b. Deleting - `RecoveryDriver::admitted` survives. -- **Merging `FeeOracleMisconfig`/`FeeOracleFatal` and - `ChainIdRpc`/`DetectionNonceRead`** — bare-string producers at - `commands/setup/mod.rs:144,171` would misattribute an operator mistake in - the black box; the second merge demotes a compile-forced classification to a - string discriminant (refuted twice on 2026-08-23). -- **A `TerminalityOf` trait for `WorkerStop`** — admits the same wrong - classification as `|_| false`, installs a crate-wide answer for `io::Error` - that `dump_info.rs:49-58` contradicts, trips `private_bounds` under - `-D warnings`, and splits an inherent convention across eight types. -- **Wiring `check-admission` into CI as one line, and pruning three - "duplicate" settle actions** — no Nix or `.envrc` is tracked and `just` is - absent from the `rust` job; TLC checks the spec against itself; the actions - are state-neutral but `InspectRetry` encodes a recorded commitment. A - properly pinned standalone `formal` job survives as a proposal. -- **A sequencer-owned `AppWithProgress` wrapper replacing the progress - capabilities** — the pair lives inside the canonical SSZ bytes the watchdog - byte-compares and the canonical machine advances it inside its own - transition; cockroach recovery reads the clock from a dump into a wiped - database. Evidence: `examples/app-core/src/wallet_snapshot.rs:41-42`, - `commands/setup/mod.rs:433-451`. Record the composition in the application - contract. **2026-09-09 refinement:** keep native progress in the engine and - expose it by value; remove the mutable accessor and capabilities. This - preserves canonical checkpoint bytes without requiring a Rust-side mirror. - See the [Application/lane review](2026-09-09-application-lane-dex-review.md). - -Also standing, from the same reviews: the **do-not-simplify list** now -lives beside the invariants it protects -([`docs/invariants.md`](../invariants.md), "Do-not-simplify"), and these -deliberate declines keep their reasons — egress single-poller fan-out (no -need at current subscriber counts), `LeaseGuard` shared release channel, -`finalized_state` ETag on `l2_tx_index` (no reachable collision), -stale-skip on-chain report (scheduler protocol change; queue behind the -scheduler library), `Storage::read` commit-vs-rollback (no behavioral -difference). - -## Historical codename map - -Older commit messages and the pre-distillation ledgers (in git history) use -these codes; their concepts now live here: - -| Code | Concept | Current home | -|---|---|---| -| R1a | write-before-broadcast watermark | I14 | -| R1b | cockroach recovery's best-effort flush | `cockroach.md` step 2 + settled above | -| R2 | content-identity check | I9, I15 | -| R3 | `synchronous=FULL` decision | `storage/open.rs` | -| R4 | exit-code contract | `commands/error.rs`, runbook | -| R5 | fail-loud check policy | invariants check policy | -| F1–F10 | 2026-06 correctness findings | settled above; F7 = the closed "WS invalidation/rollback contract" finding | -| I1–I20 | invariants (stable, still in use) | `docs/invariants.md` | -| D1–D11, H1–H14, S-A, P1–P8 | 2026-08-18 defects / harvest / structural fix / premise items | settled above + ADR | -| WP1–WP11 | 2026-06 work packages (all landed) | settled above | -| L1, L2, L3 | the lifecycle decisions (not Layer 1/2) | ADR mechanism 2 + the two August ledgers | -| S1–S7, A1–A12, B1–B5 | 2026-06 simplification queue / owed tests | open remnants above | - -## Review history - -One row per dated review ledger, oldest first. Single-pass decisions that -produced refuted entries without a ledger (the 2026-08-23 run-glue pass, the -2026-08-25 fee-oracle pass) carry their own dated blocks under Refuted. The -rows without a file were stubs whose every fact already lived here or in a -living doc; their originals are in git history, in the parent of the -distillation commit ("docs: distill the corpus — living docs timeless, -history in the register", f1b4b07): `git show f1b4b07^:docs/review/` -for `2026-06-10-correctness-review.md`, `2026-06-10-simplification.md`, -`2026-06-10-test-coverage.md`, `2026-06-25-cockroach-recovery-rooting.md`, -`2026-06-26-branch-deep-review.md`, and -`2026-08-01-containment-adr-review.md`. - -| Date | Scope | Verdict | Where its content lives | -|---|---|---|---| -| 2026-06-10 | Whole-project correctness review: twelve parallel module reviews plus line-by-line passes, every medium/high concern adversarially verified | The sequencer/scheduler duality was sound; the confirmed problems clustered at the boundary with the infrastructure underneath (fsync semantics, the local node's mempool memory, RPC fleet coherence, the subscriber protocol). Ten findings (F1–F10) and five design resolutions (R1–R5); every F-finding fixed except F7, the WS invalidation contract | R1–R5 → Settled decisions and the codename map; F7 → the open WS invalidation/rollback finding (Track 3); the robustness and hygiene backlog the review left → findings 1, 2, 3, and 6 | -| 2026-06-10 | Simplification and refactoring review (companion) | No architectural restructure: the layout is sound and is defended, not redesigned; the weight was unpinned cross-file invariants, test-only surface presenting as production API, and duplicated semantics | Created `docs/invariants.md`; the settled entry above; open remnants and declines above | -| 2026-06-10 | Test-coverage review (companion): what the suite pins, what it misses, which harness levers exist | Recovery is the best-tested subsystem (the full dispatch matrix at unit and e2e level, libfaketime clock jumps, respawn loops, TCP-proxy outage injection, Anvil mempool control); the one structural hole, the duality having no direct mechanism, is closed by the watchdog non-genesis byte-compare e2e and the I1 agreement table | Owed tests and harness levers still to build above; do not resurrect a TEST_PLAN scenario matrix | -| 2026-06-25 (follow-ups closed 2026-06-26) | Design session with an adversarial panel: how `setup --recovery` roots the rebuilt batch tree at the resume nonce | The `batch_tree_anchor` singleton: the parentless root carries the anchor nonce, exact-matched by the contiguity trigger and frozen once setup completes; no sentinel batch row | I16 and `docs/recovery/cockroach.md`; the sealed sentinel and the circular recovery-time cross-check → Refuted; the follow-up e2e's self-divergence against the empty rebuilt tree → the anchor-aware frontier on I15 | -| 2026-06-26 | Deep branch review: the setup/run split, the scheduler-library extraction, the fold engine, `setup --recovery`, plus a multi-agent adversarial sweep | Eight findings confirmed and fixed; three refuted | The two protocol contracts (`docs/protocol/scheduler-semantics.md`, `docs/protocol/application-contract.md`) were written during this review; the recovery spec is `docs/recovery/cockroach.md`; per-finding dispositions and the declined `FoldInputSource` above | -| 2026-08-01/02 | Containment ADR review: two rounds on the terminal-containment cutover, then re-evaluation with the maintainer | The architectural turn accepted; the unimplemented `LiveKernel`/reader-mailbox design rejected on the completeness/cost boundary: the content-identity check is a narrow backstop, not a divergence oracle, and SQLite remains the durable coordination plane | Every mechanism it shaped → the [authority-boundary ADR](../plans/2026-08-authority-boundary-adr.md); the divergence race bound → I15; `RunEpoch`/`EffectGate`/`LiveKernel` and the marker-file protocol → Refuted | -| 2026-08-18 | Over-engineering review: the full branch, seven parallel subsystem reviews plus an independent premise challenge of the ADR | Not over-engineered, unevenly engineered; 141 mechanisms inventoried (98 keep, 25 simplify, 6 cut, 12 question) | [`2026-08-18-over-engineering-review.md`](2026-08-18-over-engineering-review.md), kept for the inventory, the ~700-line harvest, and the eleven defects | -| 2026-08-22 | Lifecycle simplification, decision L3 | The attempt journal bought only what tracing already provided; it narrowed to the `terminal_faults` black box, and telemetry writes became verdict-neutral | [`2026-08-22-lifecycle-simplification.md`](2026-08-22-lifecycle-simplification.md), kept for the journal weight audit, the `admission.tla` ghost-variable result, and the verdict-integrity defects | -| 2026-09-03 | Branch stock-take of the authority-boundary PR: first-hand reads, then a read-only fleet of seven subsystem lenses and five premise challengers, then three refuters over the eighteen highest-ranked proposals | Proportionate overall, with three residue pockets; every proposal recorded, the jury-refuted ones listed above | [`2026-09-03-branch-stocktake.md`](2026-09-03-branch-stocktake.md), the ledger of the current branch, with its "Landed" section | -| 2026-09-07 | PR #28 premise review and maintainer-approved simplification | Ordered recovery replaces the phase driver; diagnosed terminal runtime faults abort immediately; ordinary shutdown and snapshot leases remain; reader drain race and duplicate offset check fixed | Current ADR and recovery design; finding 33 and settled decisions above. Validation: 692 host tests, seven targeted restart/outage E2Es, workspace check, strict Clippy, formatting, and admission TLC passed. The broader stale-batch recovery E2E reached its watchdog comparison but was blocked by the host Lua emulator 0.21 loading the pinned 0.20 image (archive version mismatch); no protocol pin was changed. | -| 2026-09-09 | Application, inclusion lane, and public DEX integration branch | Native progress ownership, typed validation failures, mutable independent checkpoints, optional canonical inspection, and lane bookkeeping simplified; ingress CORS and Lua 5.4 parity restored. Reference C bridge port kept separate. | [Application/lane review](2026-09-09-application-lane-dex-review.md); current Application and snapshot contracts. Workspace check, strict Clippy, 697 host tests, and 62 watchdog tests passed; private DEX conformance remains unverified. | -| 2026-09-11 | Reference C bridge port and review boundary | Keep the current Application contract, runtime payload bound, paired progress, filesystem-owned checkpoint disposal, and host failure fixes together. Engine-dependent API refinements and external-engine conformance remain follow-ups. | Settled decisions and owed tests above; [C binding guide](../protocol/c-application-binding.md), Application contract, and snapshot lifecycle. | -| 2026-09-16 | Application-history and Track 3 implementation, with independent storage/recovery/snapshot review | Replaced mixed replay and sparse attribution with current application inputs; complete atomic baselines; acceptance-derived immutable snapshots; HTTP restore/recovery archives and mandatory WS claims. Review narrowed equal-block recovery to empty genesis to avoid losing same-block pending directs. | [History design](../protocol/application-history.md), current invariants/API/snapshot/recovery docs. Validation: 693 workspace tests, strict Clippy, formatting, 62 watchdog tests, admission TLC (155 distinct states), and the Anvil recovery/old-claim refusal gate passed. Full canonical watchdog comparison remains blocked by the local emulator 0.21 vs repository 0.20 environment; no pin changed. | -| 2026-09-16 | Track 3 integration validation | Nonempty HTTP cold replica, concurrent backlog/live consumption, stale recovery/rebootstrap, and four real canonical-machine gates pass under emulator 0.20. Tooling fixes preserve Lua paths and make benchmark fee defaults admissible. | [Validation and latency evidence](2026-09-16-track3-validation.md); native bridge/DEX and representative deployment latency remain separate gates. | +# Unresolved review work + +Current follow-ups, maintained under the [review lifecycle](README.md). +Contracts and settled design reasons belong to their owners; completed review +history is in Git. Entries below were checked against code and test sources on +2026-09-17 at `85f033b0768de527312ff876a29bca67ee2f9316`. This was a source audit, +not a new run of every cited test. Recheck an entry before acting on it. + +## Confirmed discrepancies + +### Batch-size accounting omits SSZ overhead + +The lane estimates each operation as `71 + max_method_payload_bytes()`. +The SSZ layout takes `83 + actual_payload_bytes` per operation, including its +list offset, plus 12 bytes per batch and 18 per frame. Thus the estimate +understates a maximum-size operation; smaller actual payloads can mask it. +This is a batch-target accounting discrepancy, not a demonstrated protocol-size +overflow. The discrepancy is not a universal percentage. + +Evidence: [`SignedUserOp`](../../sequencer-core/src/user_op.rs), +[`Batch` / `Frame` / `WireUserOp`](../../sequencer-core/src/batch.rs), and +`user_op_count_to_bytes` in the [lane](../../sequencer/src/ingress/inclusion_lane/mod.rs). +Next: compare the intended bound with serialized batches across payload/frame +counts, then correct the estimate and check the separately configured +`batch_policy.log_user_op_bytes` used for fee accounting. + +### Setup misclassifies some deterministic L1 configuration failures + +Setup wraps reader bootstrap failures as live-worker failures, yielding exit 1; +normal-run startup classifies deterministic reader bootstrap failures as +terminal, exit 30. Malformed RPC URLs exercise this distinction. Discovery +failures such as a wrong InputBox also occur in setup, but run does not repeat +that discovery. The impact is a misleading operator hint in a one-shot command. + +Evidence: [`setup`](../../sequencer/src/commands/setup/mod.rs), +[`InputReaderError::is_terminal_invariant`](../../sequencer/src/l1/reader.rs), +and `classify_input_reader` in [startup recovery](../../sequencer/src/recovery/mod.rs). +Next: classify failures by the setup phase and pin the external exit code; +preserve genuinely transient provider failures. + +### Startup logs the full RPC URL + +The `sequencer startup` event includes `eth_rpc_url` verbatim. Operator URLs can +carry credentials in userinfo, paths, or query parameters; private-key +redaction does not cover this field. + +Evidence: [`commands/run/mod.rs`](../../sequencer/src/commands/run/mod.rs). +Next: omit the field or define a safe endpoint representation, and check +diagnostic/help paths with a synthetic credential-bearing URL. No credential +exposure in an actual deployment was established by this review. + +## Bounded investigations and cleanup + +- **Transient SQLite contention stops the submitter.** Read handles use a + 50 ms busy timeout; a storage/open failure escapes the submitter loop. + BUSY/LOCKED are nonterminal but project to unclassified exit 1, causing + respawn/recovery under a restarting supervisor. Frequency and benefit of + local retry are unmeasured. Check contention before choosing a bounded + retry or timeout change; preserve other errors. Evidence: + [`storage/open.rs`](../../sequencer/src/storage/open.rs), + [`submitter/worker.rs`](../../sequencer/src/l1/submitter/worker.rs), + [`commands/error.rs`](../../sequencer/src/commands/error.rs). +- **Canonical direct-input queue capacity.** The shared + [`Scheduler`](../../sequencer-core/src/scheduler/mod.rs) retains queued + payloads without a byte budget. Force-drain bounds age in the observed L1 + timeline, not bytes. Determine the supported L1-window volume and guest + memory cost before claiming an OOM vulnerability or proposing a limit. + Dropping or capping canonical inputs would change protocol semantics. +- **Overlapping admission policies.** The generic + [`lifecycle` preflight](../../sequencer/src/storage/lifecycle.rs) contains + setup/rebuild branches, but production calls it only for run/flush. + [`setup`](../../sequencer/src/commands/setup/mod.rs) has its own admission + path with an intentionally tested already-complete no-op. Consolidate or + narrow the unused branches when next changing admission; there is no + demonstrated conflicting live route. +- **Fee-observation visibility.** + [`log_gas_price_updated_at_ms`](../../sequencer/src/storage/fee_oracle.rs) + is persisted, with no production reader. Decide whether operator SQL + inspection suffices or a real consumer needs exposure before adding an + endpoint or removing the stamp. The accepted + [oracle outage policy](../threat-model/README.md#actors-and-trust) is an + economic tradeoff, independent of whether a health field exposes age. +- **Recovery across external effects.** Model/storage tests do not replace + process-level zombie re-injection or a restart between flush completion and + cascade commit. Before adding harness machinery, identify the missing + observation: safe nonce consumption must exclude a later original landing, + and a restarted recovery must rederive facts without reusing the previous + attempt's flush witness. Use the [recovery model](../recovery/README.md) to + bound a scenario and assess whether existing component tests suffice. + +## Verification gaps + +These are specific behaviors whose coverage remains incomplete, not a mandate +to build a general fault-injection framework. Add a discriminating assertion +at the smallest useful boundary when working on that behavior. + +| Boundary | Existing evidence and remaining check | +|---|---| +| Elapsed-time danger | Storage/procedure coverage exists. Add a process scenario isolating `EstimatedBatchInDanger` from stale-view refusal: retry at exit 20 without a speculative cascade. Start in [recovery](../../sequencer/src/recovery/mod.rs) and the [E2E scenarios](../../tests/e2e/src/test_cases.rs). | +| Canonical divergence | Storage freeze and startup refusal are covered separately. Compose accepted divergent input, runtime stop, and refusal after respawn in a process test; frontier must remain frozen. See [I15](../invariants.md#i15-divergence-marker-present--acceptance-frontier-frozen). | +| Rebuilt anchor | Anchor unit mechanics and rebuild round-trip are covered. Exercise a later full-tear cascade after a nonzero-anchor rebuild and verify submission resumes at that anchor. See [I16](../invariants.md#i16-the-batch-tree-has-exactly-one-valid-parentless-root-carrying-the-deployments-anchor-nonce). | +| Same-block directs | `multi_deposit_reconciliation_test` accumulates directs in separate blocks. Queue portal sends, mine once, assert equal receipt blocks, and verify canonical/WS order and attribution in the [E2E scenarios](../../tests/e2e/src/test_cases.rs). | +| Wallet business failure | Mixed replay is covered. Pin insufficient transfer/withdrawal amounts after successful fee validation: fee/nonce/progress advance, the transfer/withdrawal has no further effect, and replay agrees. See the [wallet implementation](../../examples/app-core/src/application/wallet.rs) and [Application contract](../protocol/application-contract.md#2-replay-safety--rejection-inclusion-and-failure). | +| Boot failure exits | Exit 20 now has a process assertion; exits 40/1 still need exact assertions in applicable setup/failure scenarios, so the supervisor receives the intended recovery/retry hint. Start in the [E2E scenarios](../../tests/e2e/src/test_cases.rs) and [exit contract](../../sequencer/src/commands/error.rs). | +| Uniswap-mode boot | Source-boundary tests pin setup validation, lazy runtime refresh, and transient quote retention. Existing sequencer E2Es use fixed mode. Decide whether a mock-pool boot scenario warrants its harness cost when changing oracle integration. | + +Warm restore already has state/cursor agreement tests and snapshot downloads +have lease/GC coverage. If restart cost becomes a requirement, add an assertion +that distinguishes restoring a dump from a correct but expensive genesis +replay; no new snapshot lifecycle is implied. + +## Integration work owned elsewhere + +- [Track 3 integration gates](../plans/2026-07-track3-feed-replay-design.md): + native snapshot-to-live replication and representative deployment latency, + including checkpoint creation and complete L1-reconciliation turns. +- [Track 6](../plans/2026-07-coordination-tracks.md#track-6--dump--application-api-redesign): + external engine/scheduler agreement, C-host end-to-end coverage, independent + fee-conversion vectors, and consumer-driven ABI/checkpoint decisions. + +The [2026-09-16 validation record](2026-09-16-track3-validation.md) supports +those integration decisions within its stated scope. It is not proof of +private-engine conformance or deployment capacity. diff --git a/sequencer-core/src/fee.rs b/sequencer-core/src/fee.rs index 81234fd9..902f1ec3 100644 --- a/sequencer-core/src/fee.rs +++ b/sequencer-core/src/fee.rs @@ -6,24 +6,14 @@ //! All fees in the protocol (frame `fee_price`, user-op `max_fee`, DB `recommended_fee`) //! are represented as **log-space exponents** with base 129/128. //! -//! An exponent `n` represents a linear value of `(129/128)^n` smallest-token-units. -//! Exponent 0 = 1 unit (minimum, effectively free). There is no special sentinel. +//! An exponent `n` nominally represents `(129/128)^n` smallest-token-units. +//! [`fee_to_linear`] owns the exact integer conversion contract, including +//! intermediate rounding and operation order. Exponent 0 is 1 unit; there is +//! no special sentinel. The encoding uses two bytes and no floating point. //! -//! This encoding: -//! - Fits any token denomination in a u16 (range up to ~10⁷⁷) -//! - Eliminates integer overflow in the DB (fee derivation becomes pure addition) -//! - Compresses fees to 2 bytes on the wire -//! - Gives ~0.78% precision per step -//! - Uses **no floating-point arithmetic** — all conversions are pure integer ops -//! -//! The key trick: multiplying by 129/128 in integer math is `x + (x >> 7)`. -//! Exponentiation uses a precomputed table of 15 entries with binary -//! exponentiation (at most 15 fixed-point multiplications). -//! -//! The precomputed table and [`MAX_EXPONENT`] are generated at build time by -//! `build.rs` using exact integer arithmetic (iterated fixed-point squaring). -//! Any reimplementation (e.g. in C++) must use the same table values to -//! guarantee bit-identical fee calculations. See `build.rs` for the algorithm. +//! `build.rs` generates the 15-entry fixed-point table and [`MAX_EXPONENT`]. +//! Independent implementations must reproduce the conversion bit-for-bit: +//! fee differences can change validation decisions and application state. use alloy_primitives::U256; @@ -51,19 +41,27 @@ type U512 = alloy_primitives::Uint<512, 8>; /// Convert a log-space fee exponent to a linear [`U256`] value. /// -/// `fee_to_linear(n)` = `floor((129/128)^n)`. +/// Let `S = 2^64`. `build.rs` generates `T[0] = 129 * 2^57` and +/// `T[i] = floor(T[i-1] * T[i-1] / S)` for `i = 1, …, 14`. /// -/// Uses a precomputed table with binary exponentiation: at most 15 fixed-point -/// multiplications, no floats. +/// Start `R = S`. Visit bits `i = 0, …, 14` in ascending order; for each set bit +/// of `n`, replace `R` with `floor(R * T[i] / S)`. Return `floor(R / S)`. +/// Every multiplication widens its 256-bit operands to a 512-bit product +/// before shifting right by 64. Intermediate flooring and accumulation order +/// are part of the contract: using the same table in a different order, or +/// computing the exact rational power and flooring only once, is not a +/// compatible replacement. /// /// # Panics /// -/// Panics if the result would overflow `U256` (exponent > [`MAX_EXPONENT`]). +/// Panics for `n > MAX_EXPONENT`. The bound protects the full fixed-point +/// accumulator, including its 64 fractional bits, rather than just the final +/// integer result. pub fn fee_to_linear(log_fee: u16) -> U256 { fee_to_linear_fixed(log_fee) >> FRAC_BITS } -/// Compute `(129/128)^n` in fixed-point representation (64 fractional bits). +/// Compute the rounded fixed-point accumulator specified by [`fee_to_linear`]. /// /// Used internally for higher-precision comparisons in binary search. fn fee_to_linear_fixed(log_fee: u16) -> U256 { @@ -82,7 +80,9 @@ fn fee_to_linear_fixed(log_fee: u16) -> U256 { /// Convert a linear fee value to the nearest log-space exponent. /// -/// `fee_from_linear(v)` = `round(log_{129/128}(v))`. +/// Choose the exponent whose rounded fixed-point value is closest to `value`, +/// breaking ties toward the smaller exponent. Values at or above +/// `fee_to_linear(MAX_EXPONENT)` saturate to [`MAX_EXPONENT`]. /// /// Returns 0 for `value <= 1` (since `(129/128)^0 = 1`). /// From b558ef54fee5653707e7697df290a7d97b459be9 Mon Sep 17 00:00:00 2001 From: gcdepaula Date: Fri, 18 Sep 2026 08:20:07 -0300 Subject: [PATCH 16/29] feat: add historical replay and checkpoint compatibility Expose era-pinned historical L1 pages and coherent history metadata for application projections. Record preserved-prefix cuts atomically with recovery and expose generation compatibility through HTTP and the Rust SDK. Cover projection restore, historical replay, missed recoveries, and subscription races. Document manual recovery readiness and separate downstream adoption from API merge criteria. --- AGENTS.md | 5 +- README.md | 127 ++++ docs/invariants.md | 9 +- docs/plans/2026-07-coordination-tracks.md | 27 +- .../2026-07-track3-feed-replay-design.md | 98 +++- docs/plans/2026-08-authority-boundary-adr.md | 6 +- docs/protocol/application-contract.md | 27 + docs/protocol/application-history.md | 53 +- docs/protocol/projection-replay.md | 122 ++++ docs/recovery/README.md | 8 +- docs/recovery/cockroach.md | 122 +++- docs/watchdog/operator-deployment.md | 22 +- sdk/rust-client/src/errors.rs | 12 + sdk/rust-client/src/history.rs | 359 ++++++++++++ sdk/rust-client/src/lib.rs | 11 +- sequencer-core/src/history_api.rs | 81 +++ sequencer-core/src/lib.rs | 1 + sequencer/src/egress/api/history.rs | 276 +++++++++ sequencer/src/egress/api/mod.rs | 6 +- .../integration_tests/historical_bootstrap.rs | 553 ++++++++++++++++++ .../recovery_compatibility.rs | 403 +++++++++++++ sequencer/src/integration_tests/mod.rs | 1 + sequencer/src/storage/egress.rs | 4 +- sequencer/src/storage/egress/historical.rs | 262 +++++++++ .../src/storage/egress/historical/tests.rs | 495 ++++++++++++++++ sequencer/src/storage/history.rs | 42 +- .../src/storage/history/generation_tests.rs | 170 ++++++ .../src/storage/migrations/0001_schema.sql | 17 +- sequencer/src/storage/mod.rs | 2 +- sequencer/src/storage/recovery.rs | 6 +- sequencer/src/storage/recovery_tests.rs | 35 ++ sequencer/src/storage/snapshot_dumps.rs | 2 +- 32 files changed, 3312 insertions(+), 52 deletions(-) create mode 100644 docs/protocol/projection-replay.md create mode 100644 sdk/rust-client/src/history.rs create mode 100644 sequencer-core/src/history_api.rs create mode 100644 sequencer/src/egress/api/history.rs create mode 100644 sequencer/src/integration_tests/historical_bootstrap.rs create mode 100644 sequencer/src/integration_tests/historical_bootstrap/recovery_compatibility.rs create mode 100644 sequencer/src/storage/egress/historical.rs create mode 100644 sequencer/src/storage/egress/historical/tests.rs create mode 100644 sequencer/src/storage/history/generation_tests.rs diff --git a/AGENTS.md b/AGENTS.md index 92b5cb78..d18eab36 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -197,7 +197,10 @@ Paths below are relative to `sequencer/src/`: mandatory offset with its source in `application_inputs`. - **History version** — `(EraId, RecoveryGeneration)`. Setup publishes a complete baseline with a fresh era; recovery increments the generation exactly once - iff it invalidates at least one valid batch. Subscription claims enforce both. + iff it invalidates at least one valid batch and records the preserved-prefix + cut in the same transaction. `/history` can check a saved checkpoint across + intervening generations; subscription claims still enforce both identifiers. + The [history contract](docs/protocol/application-history.md) owns compatibility. - **Soft confirmation** — sequencer's predicted ordering, emitted before the batch lands on L1. - **Snapshot** — immutable artifact at every batch close, registered with its local batch identity and application count. Acceptance facts select the diff --git a/README.md b/README.md index 72a83068..515989c9 100644 --- a/README.md +++ b/README.md @@ -254,6 +254,133 @@ Message shapes: { "kind": "direct_input", "offset": 11, "sender": "0x...", "block_number": 123, "block_timestamp": 1700000000, "transaction_hash": "0x...", "payload": "0x...", "input_index": 42, "batch_nonce": 4 } ``` +### History metadata and historical L1 inputs (internal only) + +Readers that maintain additional transfer/order history can reconstruct it from +L1 and then join the application feed. The +[projection replay contract](docs/protocol/projection-replay.md) describes +bootstrap, client checkpoints, pending directs, and terminal drain. + +`GET /history` returns one coherent view of the deployment, current application +history, immutable era baseline, and latest accepted checkpoint. Optional +`era_id=` requires the selected era; a mismatch returns `409 ERA_CHANGED`. +Example immediately after a rebuild: + +```json +{ + "deployment": { + "chain_id": 31337, + "app_address": "0x1111111111111111111111111111111111111111", + "input_box_address": "0x2222222222222222222222222222222222222222", + "app_deployment_block": 1, + "batch_submitter_address": "0x3333333333333333333333333333333333333333" + }, + "history": { + "version": { + "era_id": "22222222-2222-4222-8222-222222222222", + "recovery_generation": 0 + }, + "available_from": 7, + "head": 7 + }, + "baseline": { + "l1_stop_block": 1240, + "l1_end_input_index": 8, + "next_batch_nonce": 2 + }, + "accepted_checkpoint": null, + "compatibility": null +} +``` + +- `history.available_from` is baseline application count `K`; entries `[K,head)` + are available through WS. Counts include all executed application inputs. +- `baseline` describes the fixed L1 stopping block `C`, exclusive InputBox end + `R`, and scheduler nonce after recovery's terminal drain. It survives generation + changes and baseline artifact GC. It is distinct from the moving safe head. +- `accepted_checkpoint`, when available, has `inclusion_block`, + `executed_input_count`, and `next_batch_nonce`, under `history.version`. + Genesis supplies the zero checkpoint; a rebuilt baseline is not itself an + accepted checkpoint. The metadata does not lease or download a native artifact + and does not certify a client projection. A known divergence returns `503`. +- `compatibility` is `null` unless `from_generation=` is supplied together + with `era_id`. It then contains `from_generation` and `preserved_input_count`: + the prefix that survived every standard recovery since that generation, + bounded by the current head. A future generation or missing era returns + `400 BAD_REQUEST`; an era mismatch takes precedence over the generation bound. + +For example, `GET /history?era_id=&from_generation=0` can return +`"compatibility": {"from_generation": 0, "preserved_input_count": 3}`. +A saved checkpoint from that era/generation is reusable when its count `X` +satisfies `K <= X <= 3`. The boundary is inclusive: the checkpoint has executed +entries before `X`, and resumes at entry `X`. Each checkpoint must be checked +using its own saved generation. With no intervening recovery, the bound is the +current head. The [history contract](docs/protocol/application-history.md#checkpoint-compatibility-after-standard-recovery) +defines the calculation and trust boundary. + +Restore an eligible checkpoint, persist the response's current history version +with it, and subscribe using that version and its actual count. A recovery +between lookup and subscription still returns `STALE_GENERATION`; repeat the +lookup using the version associated with the restored state. Compatibility does +not certify the client's application or projection implementation, and cannot +cross a cockroach recovery's new era. + +`GET /historical-l1-inputs` requires `era_id` and exactly one starting selector: + +- `next_input_index=`: inclusive per-application InputBox index, starting at 0. +- `after_block=`: initially seek to the first input strictly after that block; + continue using the returned `next_input_index`. + +The endpoint serves only `[0,R)` through the selected era's `C`. A response to +`next_input_index=5&limit=1` can be: + +```json +{ + "era_id": "22222222-2222-4222-8222-222222222222", + "l1_stop_block": 1240, + "end_input_index": 8, + "next_input_index": 6, + "items": [{ + "input_index": 5, + "sender": "0x3333333333333333333333333333333333333333", + "payload": "0x00", + "block_number": 1230, + "block_timestamp": 1700014760, + "transaction_hash": "0x4444444444444444444444444444444444444444444444444444444444444444" + }] +} +``` + +Records preserve original inner payloads and authenticated senders, including +malformed/rejected batches; they are not complete `EvmAdvance` envelopes. Indices +are contiguous and ordered. Binary values are hex; timestamps are Unix seconds. +Clients must preserve integer precision. A page may split a block. + +Optional `limit` defaults to 256 and accepts 1–256. Pages target 1 MiB of raw +payloads; a larger first input is returned alone, intact. Hex encoding increases +wire size, so this is not a hard response-size limit. Eight historical responses +can be in flight; a permit remains held through body delivery or cancellation. +SQLite read transactions end before network delivery. These limits bound memory +by the page target or largest single input, not total history length. + +Only `next_input_index == end_input_index` means EOF; a short page does not. +Requesting `next_input_index=R` or `after_block=C` returns an empty completed page. +Generation changes do not invalidate historical pages; an era change does. + +Malformed/unknown query fields, invalid selectors/limits, or positions above +`R`/`C` return the existing `400 BAD_REQUEST` JSON shape. An era mismatch returns +the existing `409 ERA_CHANGED` history-policy body before semantic position +checks. Capacity exhaustion returns `429 OVERLOADED`; shutdown or an operational +read failure returns `503 UNAVAILABLE`. Interrupted bodies are failed pages. +Missing durable rows or other storage invariant failures follow the process's +terminal fault policy, never a successful partial page. + +The Rust SDK exposes `history(expected_era, from_generation)` and +`historical_l1_inputs(era, start, limit)` with typed metadata and era refusals. +Both use the configured request deadline, including body transfer; callers may +increase it for large historical inputs. The client owns replay, persistence, +checkpoint selection, and subscription. + ### Operator snapshot endpoints (internal only) These serve application state to the operator's watchdog and indexers. diff --git a/docs/invariants.md b/docs/invariants.md index 422aafff..5a9b6c6e 100644 --- a/docs/invariants.md +++ b/docs/invariants.md @@ -81,7 +81,7 @@ by writer and are write-once (`0001_schema.sql`). | inclusion lane | `batches` (insert + `sealed_at_ms`), `frames`, `user_ops`, `application_inputs`, `dumps`/`snapshots` (batch close) | | input reader | `safe_inputs`, `l1_safe_head`, `safe_accepted_batches`, `canonical_divergence` (the divergence poison marker) | | recovery (startup) | `batches.invalidated_at_ms`, Tip reopen, current `application_inputs` suffix deletion | -| history metadata (setup/recovery) | `history_state` — complete era/application-count/L1-block baseline, generation bump in a non-empty standard-recovery cascade | +| history metadata (setup/recovery) | `history_state` — complete era/application-count/L1-block baseline; generation advance and immutable preserved-prefix cut in a non-empty standard-recovery cascade | | batch submitter and mempool flusher | `wallet_nonce_watermark` — deliberately shared under one protocol: each raises it before its first broadcast (write-before-broadcast, I14) | | egress (HTTP) | `dumps.lease_count` (leases); `run`'s startup hygiene resets it to zero as the crash backstop | | setup | `deployment_identity` (pinned once), `batch_tree_anchor` (the root nonce, frozen once setup completes), the initial `dumps` + `snapshots` rows (genesis or rebuild registration, atomic with the complete history baseline), the `setup_complete` fact (written once), `batch_policy.log_gas_price` + `log_gas_price_updated_at_ms` (first write; Fixed and Uniswap) | @@ -160,7 +160,7 @@ by writer and are write-once (`0001_schema.sql`). ### I5. Recovery removes exactly the invalidated application suffix - **Holds:** invalidating a batch deletes its `application_inputs` through the - schema trigger. The cascade, generation increment, and replacement Tip commit + schema trigger. The cascade, generation cut/increment, and replacement Tip commit together. Original source records and immutable snapshots remain; snapshot selection excludes invalidated batches and GC retires their unleased artifacts. - **Enforced by:** `cascade_and_reopen`, application-input constraints, valid views. @@ -425,7 +425,10 @@ by writer and are write-once (`0001_schema.sql`). transaction. The history row is absent before this boundary. `K` and `C` remain immutable even after baseline artifact GC or recovery-root invalidation. - **Standard recovery:** one generation increment iff a valid batch is - invalidated, in the cascade transaction. Clean restart changes neither token. + invalidated, with an immutable cut at the count after suffix deletion and + before replacement directs. The entire transition commits in the cascade + transaction. Clean restart changes neither token. Every intervening cut is + required to authorize reusing a checkpoint from an older generation. - **Enforced by:** `complete_baseline_setup`, immutable history triggers, exact-`+1` generation trigger, and `cascade_and_reopen`. - **Depended on by:** mandatory snapshot-derived WS claims. Identity is validated diff --git a/docs/plans/2026-07-coordination-tracks.md b/docs/plans/2026-07-coordination-tracks.md index d10ffd43..6158aa1d 100644 --- a/docs/plans/2026-07-coordination-tracks.md +++ b/docs/plans/2026-07-coordination-tracks.md @@ -12,14 +12,14 @@ freely at this stage — no backward-compatibility constraints. | # | Track | Owner | Status | |---|-------|-------|--------| -| 3 | Feed & replay protocol redesign | us (design) → us/Stephen (impl) | **implemented** — canonical application history, snapshot restore archives, mandatory WS claims, typed refusals, and SDK cutover; [remaining integration gates](2026-07-track3-feed-replay-design.md#remaining-integration-gates) | +| 3 | Feed & replay protocol redesign | us (design) → us/Stephen (impl) | **repository API implemented; post-merge adoption and deployment work remain** — [follow-up sequence](2026-07-track3-feed-replay-design.md#follow-up-sequence) and [ownership](2026-07-track3-feed-replay-design.md#merge-scope-and-follow-up-ownership) | | 5 | Fee exponentiation LUT | us | **deferred** — decided exact-floor if built (the table *is* the spec, algorithm-free; replay continuity across the upgrade explicitly not preserved); a separate pending design decision may make log-space fees defunct — revisit after syncing with Bart | | 6 | Dump / `Application` API redesign | us + Bart | **interface and reference C binding implemented** — [Application contract](../protocol/application-contract.md); native-engine integration gates remain | **Current campaign order:** -1. Validate snapshot-to-live replica bootstrap through the reference C bridge, then the private DEX engine when shared. -2. Remeasure feed latency in the representative environment. +1. Merge the implemented egress API after repository review/checks. Bart can then integrate his client; adjust the API from concrete feedback without waiting for downstream completion. +2. Extend reference C-bridge coverage in this repository. Application integrators/operators own private-engine validation, the canonical-to-native exporter and recovery drill, and representative capacity measurements before production use. 3. Track 5 (fee LUT) only after the log-space-fees decision. Full restore archives now support file and directory application prefixes. @@ -33,9 +33,15 @@ bootstrap, history identity, replay, and recovery boundaries. The and canonical recovery/watchdog gates have a [validation record](../review/2026-09-16-track3-validation.md). -Remaining work is native-engine integration and representative deployment -latency, tracked in the [integration plan](2026-07-track3-feed-replay-design.md). -Additional transport or retention mechanisms require a measured consumer need. +Readers whose projections contain information absent from the latest application +state can use the implemented fixed-prefix historical L1 API and checkpoint +metadata. The [projection contract](../protocol/projection-replay.md) owns that +workflow; the history contract owns implemented checkpoint compatibility across +standard recoveries. The [integration plan](2026-07-track3-feed-replay-design.md) +owns follow-up requirements and their owners. Native-engine integration and +representative deployment latency remain open after merge; they are not egress +API merge prerequisites. Other transport or retention mechanisms require a +measured need. ## Track 5 — Fee exponentiation LUT (deferred) @@ -64,6 +70,15 @@ conformance cannot establish private-engine correctness. Remaining checks need the actual consumer: +- Supply the application's versioned canonical-machine-to-native recovery + exporter and completed operator runbook. Require the + [non-genesis recovery drill](../recovery/cockroach.md#recovery-readiness-before-deployment) + for production readiness: the old native state is unavailable, the exported + bundle restores correctly, and execution after rebuild matches the canonical + machine. For the DEX, pin the designated state drive/memory region and derive + resume metadata from canonical execution. Add the integration check to the + release validation once the actual artifacts are available; no generic trait + or deployment gate currently enforces this requirement. - Exercise snapshot-to-live bootstrap and canonical comparison through the C host in CI; its current smoke test builds and invokes `--help`. A reusable conformance runner needs engine-supplied genesis and meaningful accepted and diff --git a/docs/plans/2026-07-track3-feed-replay-design.md b/docs/plans/2026-07-track3-feed-replay-design.md index 6a2700d7..6551f8fd 100644 --- a/docs/plans/2026-07-track3-feed-replay-design.md +++ b/docs/plans/2026-07-track3-feed-replay-design.md @@ -8,24 +8,104 @@ The application-history protocol is implemented. Its current contracts live in: - [Snapshot lifecycle](../snapshots/lifecycle.md): durable artifacts, accepted comparisons, recovery exports, and leases. -## Remaining integration gates +## Merge scope and follow-up ownership -1. Validate the native reference adapter's snapshot-to-live replica workflow; - repeat against the private DEX bridge when available. Reference-engine - conformance does not establish private-engine correctness. -2. Measure submit-to-matching-WS-event latency in the representative deployment, - including checkpoint creation and L1 reconciliation under the supported load. +The repository delivery includes the egress contracts, storage, SDK, and reference +replay/recovery tests. Its merge criteria are review and the relevant repository +checks. Private-engine integration, operational rehearsal, and representative +capacity measurements are follow-up work; they do not block merging this API. +Merge the implementation, let consumers integrate, and adjust the API from +concrete feedback. There are no live deployments requiring compatibility. + +| Follow-up | Owner | When it is needed | +|---|---|---| +| Native reference adapter snapshot-to-live and recovery coverage | Sequencer maintainers | Additional repository conformance coverage after merge; the wallet projection tests already exercise this API's replay and recovery contract. | +| Private DEX scheduler, indexer, and database backups | Bart / application integration | After merge, while adopting the API. Verify complete checkpoint/claim association and scheduler replay; report missing fields or awkward workflow for adjustment. | +| Canonical-to-native export and incident rehearsal | Application integration and operators, under Track 6 | Before relying on that application's recovery procedure in production. | +| Ingress latency, indexing headroom, and recovery capacity | Sequencer/application maintainers and deployment operators | Before claiming support for the target deployment workload; measure historical serving alongside ordinary traffic. | The [validation record](../review/2026-09-16-track3-validation.md) records the wallet's nonempty cold bootstrap, concurrent replay/live delivery, recovery and rebootstrap, canonical-machine gates, and local latency measurements. Those -results do not replace the consumer/environment gates above. +results do not establish private-engine conformance or target-deployment capacity. + +## Application projections and recovery + +Historical bootstrap and standard-recovery checkpoint reuse are implemented. +Bounded internal readers can keep additional +application-specific transfers, orders, deals, and portfolio history outside the +sequencer's application state. The client owns indexing, complete checkpoints, +and replay. The sequencer owns optimistic ordering; the scheduler remains the +canonical authority. + +Current contracts live in the [projection replay guide](../protocol/projection-replay.md) +and [README API](../../README.md#history-metadata-and-historical-l1-inputs-internal-only). +`/history` supplies deployment/baseline/current-generation metadata and a coherent +accepted checkpoint receipt. `/historical-l1-inputs` serves the immutable raw +prefix through the era's stop block, with block seek, bounded pages, and typed +era refusal. `/history` also checks a saved generation against every intervening +recovery cut. The SDK exposes both reads. Standard recovery records cuts in its +existing transaction before replacement directs are inserted. + +The [reference integration test](../../sequencer/src/integration_tests/historical_bootstrap.rs) +restores a complete wallet/projection checkpoint, replays one-record HTTP pages +through the scheduler, exercises a malformed-batch overdue drain and terminal +drain, and subscribes at nonzero `K`. It checks application state and explicit +projection order, including a same-block pending direct and the first live +input. Storage/API/SDK tests cover limits, oversized single inputs, fixed prefix +boundaries, era/generation behavior, errors, and response deadlines. This is +reference evidence, not private-engine conformance or a deployment benchmark. + +The [checkpoint compatibility test](../../sequencer/src/integration_tests/historical_bootstrap/recovery_compatibility.rs) +uses guarded recovery, complete wallet/projection backups, HTTP compatibility +lookups, and WS replay. It distinguishes equal counts from different generations, +restores the nearest eligible checkpoint after missed recoveries, and retries a +recovery between lookup and subscription. Storage tests cover cuts before +replacement directs, empty/no-op invalidation, nonzero baselines, and rollback. + +### Goals and acceptance criteria + +| Goal | Status / remaining outcome | +|---|---| +| Historical bootstrap | Implemented: replay the complete fixed prefix and join the feed at `K`, including after a nonzero rebuild. | +| Standard recovery | Implemented: choose the nearest retained checkpoint whose prefix survived every intervening generation change, then restore/resubscribe. | +| Cockroach reader recovery | Core replay/metadata workflow and reference checkpoint restore implemented; Bart's actual checkpoint preparation, incident validation, and fallback rehearsal remain. | +| Stable coordinates | Existing era/generation/application count for claims; separate InputBox indices for raw paging. Counts alone do not certify cross-era compatibility. | +| Bounded serving cost | Page/item/response bounds implemented; representative bootstrap must preserve the ingress latency target and demonstrate catch-up headroom. | +| Operational readiness | Each production application supplies its canonical-to-native exporter, runbook, and non-genesis recovery drill under Track 6. | + +### Follow-up sequence + +1. **Consumer adoption after merge.** Bart integrates accepted-boundary checkpoint + preparation and scheduler replay using the implemented API. Sequencer + maintainers address concrete feedback as it arrives; downstream completion + is not a prerequisite for repository delivery. Before production use, rehearse + identifying an unsound projection checkpoint, including one below new `K`, + and restoring an earlier trusted backup or genesis. The API does not certify + the client's projection. +2. **Repository conformance and deployment readiness.** Extend native snapshot-to-live + validation and measure latency, historical serving cost, projection throughput, + catch-up headroom, and recovery time. Track 6 independently owns the versioned + canonical-machine exporter and native-state-unavailable drill. + +### Scope boundaries + +Keep the existing recovery terminal drain: the first accepted resumed frame +accounts for old pending directs before its user ops. Replacing that mechanism +would require carrying pending work across the baseline and is not justified by +this consumer requirement. + +Keep manual cross-era trust selection. Execution-prefix hash chains can identify +an input trace but cannot prove that an engine or indexer computed correct state; +revisit only if automated matching or measured reconstruction costs justify them. +Server-side scheduler replay producing a flattened execution archive and +submitter/key rotation remain separate future work. No client checkpoint +registration, server-side projection storage, or historical execution archive is +required by this design. ## Revisit only with a consumer need - Resumable snapshot transfer: when artifact size makes interrupted downloads costly. -- Retained client checkpoints: when full rebootstrap cost matters. -- Archival HTTP replay or raw L1 feeds: for an identified consumer. - Session fencing: if history can mutate within an admitted process or multiple local writers become supported. diff --git a/docs/plans/2026-08-authority-boundary-adr.md b/docs/plans/2026-08-authority-boundary-adr.md index b2e8672e..bc1b4ce5 100644 --- a/docs/plans/2026-08-authority-boundary-adr.md +++ b/docs/plans/2026-08-authority-boundary-adr.md @@ -179,7 +179,7 @@ clean restart changes neither. The pair is an equality/discontinuity token, not an ordered counter. Snapshot headers and mandatory WS claims expose these coordinates. Every application row has its pre-execution count; recovery replaces only the current suffix. See the [history contract](../protocol/application-history.md) and -[remaining integration gates](2026-07-track3-feed-replay-design.md). +[follow-up plan](2026-07-track3-feed-replay-design.md). ## Performance posture @@ -189,5 +189,5 @@ the evaluation conditions. The [retained comparison](../review/2026-09-16-track3 records exact revisions, workload, and same-host ACK/WS measurements. They are regression evidence: client/host contention and excluded startup or backlog work prevent interpreting them as deployment capacity. Representative latency, -including checkpoint and L1-reconciliation overlap, remains an -[integration gate](2026-07-track3-feed-replay-design.md#remaining-integration-gates). +including checkpoint and L1-reconciliation overlap, remains a +[deployment follow-up](2026-07-track3-feed-replay-design.md#merge-scope-and-follow-up-ownership). diff --git a/docs/protocol/application-contract.md b/docs/protocol/application-contract.md index 536a69ab..3bc28a28 100644 --- a/docs/protocol/application-contract.md +++ b/docs/protocol/application-contract.md @@ -193,3 +193,30 @@ implementation. `CanonicalState::canonical_snapshot_bytes` is a separate inspection trait required by the shared Rust scheduler's inspection method and canonical harness, not by the native sequencer. Human-readable debugging state also stays on the concrete application. + +### 7. Canonical recovery integration + +Every supported production application must provide a reproducible mapping from +a trusted canonical machine checkpoint and pinned deployment configuration to +its native recovery artifact. All logical state needed for future application +execution, including progress, must be recoverable this way. Native caches and +backing resources may be reconstructed; indispensable mutable application state +cannot exist only in the sequencer's local storage. + +For the native DEX integration, the native application state is the designated +canonical machine drive/memory region. The integration must pin its location, +layout, and extraction procedure for each supported image. Other applications +may require a different mapping. The recovery bundle also needs the exact L1 +boundary and next scheduler nonce, obtained from trusted canonical execution; +these are separate from merely extracting application bytes. + +Each integration supplies a versioned export command and operator procedure, +and demonstrates recovery from a non-genesis canonical checkpoint before +production use. The [recovery readiness requirements](../recovery/cockroach.md#recovery-readiness-before-deployment) +own the procedure and drill. This is an integration/release requirement, not a +claim that the private engine or current tooling has passed it. + +Canonical extraction belongs to application-specific recovery tooling. The +runtime `Application` trait stays independent of machine image layouts and host +export tooling; requiring a method to compile would establish its availability, +not the correctness or operational readiness of the recovery path. diff --git a/docs/protocol/application-history.md b/docs/protocol/application-history.md index 20232418..ed3e3bff 100644 --- a/docs/protocol/application-history.md +++ b/docs/protocol/application-history.md @@ -55,7 +55,9 @@ If the era baseline count is `K` and the current head is `H`, available entries occupy `[K, H)`. A claim at `H` waits for future entries; a claim below `K` or above `H` is refused. Identity is checked before position. Equal counts cannot authorize resuming a different era or generation, even if the consumer believes -its state precedes the replaced suffix. +its state precedes the replaced suffix. The compatibility query below can +authorize rebinding a surviving checkpoint to the current version; WS itself +continues to require exact identity. Automatic recovery invalidates a batch suffix, removes its current application rows, advances the generation, and opens the replacement Tip in one transaction. @@ -65,6 +67,40 @@ sequence is not separately retained. A repair that invalidates nothing leaves the generation unchanged. The [recovery guide](../recovery/README.md) owns repair selection and guards. +### Checkpoint compatibility after standard recovery + +Each generation transition records the surviving application count after the +invalidated rows are removed and before the replacement Tip adds any directs. +This cut commits atomically with invalidation, the generation advance, and +reopening. An invalidated empty batch still creates a transition with the old +head as its cut; a repair that invalidates nothing creates neither. The cuts are +immutable and retained for the era's lifetime. + +For a checkpoint saved in generation `g`, `/history` returns the current version +at generation `G`, head `H`, and preserved count: + +```text +P = min(H, cut[g+1], ..., cut[G]) +``` + +For `g = G`, `P = H`. A checkpoint at count `X` is eligible to resume under the +returned version exactly when `K <= X <= P`. Check each saved checkpoint using +its own era and generation, then choose the newest eligible one. For cuts +`0 -> 1: 3` and `1 -> 2: 5`, a generation-0 checkpoint at 4 is invalid, while a +generation-1 checkpoint at 4 is eligible. Looking only at the latest cut would +incorrectly reuse the former. Cuts and current history are read together; a +missing intervening transition is an invariant failure, not permission to take +the minimum over an incomplete ledger. + +The client owns checkpoint consistency: application state, projection, and claim +must describe the same executed prefix. Once compatibility is established, +persist the new version with the restored checkpoint before continuing. If +another recovery wins the race with subscription, query again using that saved +version. A stale response cannot weaken WS admission. The query proves prefix +preservation under standard recovery's trusted local bookkeeping; it does not +inspect client state or establish trust after a software bug. A new era requires +the [manual projection recovery procedure](projection-replay.md#client-checkpoints). + ### Era baseline Setup publishes a complete baseline only after its artifact is durable: @@ -81,13 +117,14 @@ Genesis supplies the trusted block-zero comparison state. The [rebuild guide](../recovery/cockroach.md) owns checkpoint requirements and the fixed stopping boundary. -## Three consumers of checkpoints +## Consumers of checkpoints | Consumer | Starting point and continuation | |---|---| | Native restart | Load the newest surviving batch snapshot, or baseline, check the engine's count against its row, then replay current application inputs. | | Sequencer replica | Download `/latest_snapshot`, restore its application state, and subscribe using the matching history claim. This follows optimistic execution. | | Watchdog | Start from independently trusted canonical machine state and replay L1. Compare at the sequencer's accepted checkpoint; the replica feed does not establish independent trust. | +| Application projection | Reconstruct additional transfer/order history using the era's historical L1 prefix, then join the application feed at the immutable baseline. The [projection replay contract](projection-replay.md) owns its checkpoint preparation and handoff. | A batch-close snapshot is identified by its local batch identity, not just its count or nonce. Recovery can reuse a nonce and empty batches can repeat a count. @@ -109,19 +146,22 @@ explains block-boundary comparison and rollback retention. The 3. Subscribe with the matching era, generation, and next-input count. Snapshot selection, headers, and lease share one transaction; recovery during the download can still invalidate the claim before subscription. Rebootstrap - if the server refuses that old identity. + if the server refuses that old era; within the same era, a compatible saved + checkpoint can instead be selected through `/history`. 4. Require each entry's offset to equal the application's current count, then execute it through the shared execution boundary. Successful application advances the count by one. Persist the history identity with the replica's state so that a later resume cannot combine different histories. 5. After an ordinary disconnect, reconnect with that saved identity and the - actual count. A history mismatch or unavailable prefix requires a current - snapshot. A count ahead of the server's head is an invalid claim to correct. + actual count. A generation mismatch permits the compatibility procedure + above. An unavailable prefix or lack of a compatible checkpoint requires a + current snapshot. A count ahead of the server's head is an invalid claim to correct. The [Rust SDK](../../sdk/rust-client/src/lib.rs) returns a `HistoryClaim` with its snapshot response and requires an explicit claim for subscriptions. The consumer owns restore, persistence, and reconnect. Fetching fresh identity -metadata cannot authorize old application state. +metadata alone cannot authorize old application state; an explicit compatibility +result can authorize a saved prefix within the same era. One durable page reader handles both backlog and live delivery, so there is no separate cursor to switch at the live boundary. Each page reads identity, @@ -144,6 +184,7 @@ writers become supported. | Coherent application pages | [`storage/egress/canonical.rs`](../../sequencer/src/storage/egress/canonical.rs) and its tests — source context, nonzero baselines, replacement offsets, concurrent recovery, gaps and SQL limits. | | Replay followed by live delivery | [`l2_tx_feed`](../../sequencer/src/egress/l2_tx_feed/) — bounded deep replay, identity refusals, shutdown and persistent faults; [`catch_up.rs`](../../sequencer/src/ingress/inclusion_lane/catch_up.rs) for native replay. | | Artifact and claim association | [`snapshot_endpoints.rs`](../../sequencer/src/integration_tests/snapshot_endpoints.rs) — headers, restore, archive contents, and lease lifetime. | +| Checkpoint compatibility | [`storage/history.rs`](../../sequencer/src/storage/history.rs) and recovery tests — immutable cuts, complete lineage, and transaction rollback; [`recovery_compatibility.rs`](../../sequencer/src/integration_tests/historical_bootstrap/recovery_compatibility.rs) — saved projections across missed recoveries and HTTP lookup/WS admission races. | The [integration validation record](../review/2026-09-16-track3-validation.md) records wallet replica and canonical-machine evidence. Remaining consumer and diff --git a/docs/protocol/projection-replay.md b/docs/protocol/projection-replay.md new file mode 100644 index 00000000..33f582f6 --- /dev/null +++ b/docs/protocol/projection-replay.md @@ -0,0 +1,122 @@ +# Application projections from historical L1 inputs + +A reader may maintain transfers, orders, or other application-specific history +absent from the latest application state. It owns that projection and its +checkpoints. The sequencer supplies raw scheduler inputs for the immutable era +baseline, then application inputs for the optimistic suffix. The canonical +scheduler remains the ordering authority. The [README API](../../README.md#history-metadata-and-historical-l1-inputs-internal-only) +owns routes, fields, limits, and refusals; the +[history contract](application-history.md) owns application coordinates. + +## Bootstrap and handoff + +1. Read `/history`. Pin the deployment identity and era, baseline application + count `K`, L1 stop block `C`, exclusive raw end `R`, and next scheduler nonce. + Use the deployment's application/genesis configuration and scheduler release, + including its batch codec, wait bound, and signing-domain name/version. +2. From trusted genesis, request historical records from raw index 0 and process + every record through the scheduler. Preserve InputBox order, including within + blocks. Malformed/rejected batch payloads still reach the scheduler: its + overdue-direct backstop runs before batch decoding. +3. Continue using the returned raw cursor until it equals `R`; page length and + block changes are not EOF. Drain the remaining directs through `C`, reproducing + the recovery terminal drain. Verify resulting count `K` and next nonce. +4. Refresh `/history?era_id=` for the current generation, confirm + the baseline association, and subscribe at application count `K`. The feed + contains entries `[K,head)` and then future entries. It does not redeliver the + inputs already included in the baseline. + +Raw records carry original inner payloads and authenticated sender/block context, +not the complete machine transport envelope. The signing domain uses the pinned +release plus deployment chain id and app address. Timestamps and transaction +hashes are provenance, not additional scheduler transition inputs. + +An ordinary generation change leaves `C/R/K/nonce` and historical pages intact. +If it occurs before subscription, refresh the same era's metadata and retry the +claim. An era change requires establishing correspondence with the new baseline; +never silently splice its pages into an earlier replay. Count/nonce equality is +a consistency check, not independent proof that a client computed correct state. + +The terminal-drained baseline need not equal canonical state at block `C`: +young directs may execute preemptively. A later accepted batch provides the +canonical comparison boundary. Keep that distinction when validating recovery. + +## Client checkpoints + +Save core application state, projection, and their actual `HistoryClaim` +consistently. An exact era-baseline checkpoint can be rebound to the current +generation after confirming the same immutable era/baseline. For suffix +checkpoints, query `/history?era_id=&from_generation=` +and require `K <= saved_count <= compatibility.preserved_input_count`. Query each +candidate using its own generation and choose the newest eligible backup. +Restore the complete application/projection checkpoint, persist the returned +version with it, and resume at its saved count. Repeat compatibility lookup if +another recovery causes WS to refuse that version. The +[history contract](application-history.md#checkpoint-compatibility-after-standard-recovery) +owns the calculation and its standard-recovery trust assumptions. + +For efficient manual recovery, prepare checkpoints at supported accepted L1 +block boundaries. Read the coherent `accepted_checkpoint` and `history.version` +from `/history`; restore an earlier compatible client backup and replay the +application feed exactly to the accepted count. Save the complete result with +that receipt. If the live reader is already ahead, use an independent restore; +the client owns the copy/replay cost. Bind the receipt to the matching history +and count. Count alone does not identify its block/nonce: empty batches can +share a count, and generations can reuse replaced offsets. + +Such a backup contains core state and projection at count `X`, inclusion block +`B`, next scheduler nonce `N`, and the application's own clock `A`. The +[manual recovery contract](../recovery/cockroach.md#replay-boundaries) requires +`A < B`, except known empty genesis, and `B <= C` for the target rebuild: + +1. Independently establish trust in the backup, projection implementation, and + checkpoint boundary under the [incident playbook](../recovery/cockroach.md#application-specific-reader-state). +2. Fetch raw records **after `A`**, not after `B`. Enqueue external directs from + `(A,B]` without executing them; skip batch envelopes in this seed range. +3. Process all records in `(B,C]` through the scheduler at nonce `N`, then perform + the same terminal drain and handoff as genesis bootstrap. + +Seeds and replay may share one paginated traversal. Page boundaries can split +block `B`; they must not cause premature replay or draining. An arbitrary +mid-batch optimistic checkpoint lacks this scheduler continuation. If an eligible +backup cannot be trusted, use an earlier eligible backup or trusted genesis. +Watchdog agreement on current application bytes does not certify the reader's +additional transfer/order history. + +If checkpointing during raw replay, persist the scheduler queue/nonce and raw +cursor alongside app/projection state, or resume from the prepared backup. +Persisting only an application count cannot resume an interrupted scheduler. + +## Worked recovery boundary + +With the 1200-block wait bound, consider valid direct inputs `D*` and accepted +user operations `U*` in this raw order: + +| Raw index | Block | Input | Application execution | +|---:|---:|---|---| +| 0 | 5 | `D0` | Queued | +| 1 | 12 | `D1` | Queued | +| 2 | 20 | Batch 0, safe block 10, `U0` | `D0`, `U0` | +| 3 | 20 | `D2`, after the batch | Queued | +| 4 | 24 | `D3` | Queued | +| 5 | 1230 | Malformed batch | Backstop executes `D1,D2,D3`; decoding rejects | +| 6 | 1232 | Batch 1, safe block 1228, `U1` | `U1` | +| 7 | 1235 | `D4` | Queued | + +The backup at `B=20` has count 2, clock `A=10`, and next nonce 1. A count-1 +backup sits partway through batch 0; it must execute `U0` before attaching this +receipt. Seed reconstruction starts at raw index 1, preserving both `D1` and the +same-block `D2` without executing batch 0 twice. + +For a rebuilt stop `C=1240`, the malformed input executes three overdue directs +without consuming a batch nonce. Batch 1 executes `U1`; terminal drain executes +`D4`. The resulting baseline is `K=7`, next nonce 2, raw end `R=8`, and app clock +1235. Subscribe at application offset 7; the next raw index is independently 8. + +The [reference integration test](../../sequencer/src/integration_tests/historical_bootstrap.rs) +restores a wallet checkpoint with separately saved notice history, deletes the +source backup, fetches one-record HTTP pages through the SDK, executes this +trace, and joins the real WS feed at nonzero `K`. It checks full wallet state and +explicit projection order against uninterrupted execution. It uses controlled +storage fixtures and a trusted checkpoint receipt; it does not establish Bart's +private database backup procedure or canonical-machine export conformance. diff --git a/docs/recovery/README.md b/docs/recovery/README.md index a1bdf1e2..1a45fa52 100644 --- a/docs/recovery/README.md +++ b/docs/recovery/README.md @@ -223,7 +223,13 @@ wall-clock aging alone can change the decision after inspection. Invalidation, history rewind, generation change, and Tip creation share one transaction. Invalidation removes the suffix's `application_inputs` projection; raw source facts remain. `RecoveryGeneration` increments once iff at least one -valid batch was invalidated. Failed reopening rolls all of this back. +valid batch was invalidated. Each advance appends an immutable generation cut: +the surviving count after suffix deletion, before reopening can insert any +replacement directs. Failed reopening rolls all of this back. The cuts let +readers determine whether a saved checkpoint survived several recoveries; +the [history contract](../protocol/application-history.md#checkpoint-compatibility-after-standard-recovery) +owns that query. Invalidating an empty batch records the old head; a no-op repair +records no transition. The new Tip follows the latest surviving batch, or uses the immutable root anchor if none survives. It attributes direct inputs after the surviving diff --git a/docs/recovery/cockroach.md b/docs/recovery/cockroach.md index ff588f73..38d07d15 100644 --- a/docs/recovery/cockroach.md +++ b/docs/recovery/cockroach.md @@ -30,13 +30,125 @@ comparison checkpoint at that block. [Standard recovery](README.md) instead uses the existing database to repair an optimistic suffix automatically. It assumes the local bookkeeping is trustworthy. +## Recovery readiness before deployment + +Canonical-to-native recovery is a required +[application integration capability](../protocol/application-contract.md#7-canonical-recovery-integration). +Each production deployment must have a release-matched export command, a +completed application-specific runbook, and a successful recovery rehearsal. +An incident is an execution of that prepared procedure, not the first attempt +to determine a state layout or assemble recovery metadata. + +The application runbook must name: + +- The canonical image, native engine, state layout, exporter, and tool versions. +- How to select and preserve a trusted CM checkpoint and its exact L1 boundary. +- The command extracting the application state, count/clock, and next scheduler + nonce into the bundle accepted by `setup --recovery`, including supported + checkpoint boundaries and the loader's `A < B` requirement below. +- Artifact locations, access/backup procedures, validation commands, and the + commands to rebuild, restart, compare, and resume affected readers. +- A rehearsed fallback to an earlier trusted checkpoint or genesis if the + preferred artifact is unavailable, with measured replay time and disk needs. + +The qualifying drill starts with a **non-genesis canonical machine checkpoint**, +pinned deployment data, and L1 access, while the old native database and dumps +are unavailable. Run the actual exporter and restore its output; check the +native application bytes/progress and scheduler nonce against the canonical +source. Exercise directs pending at the checkpoint and inputs arriving after it. +Run `setup --recovery` in a fresh directory, resume sequencing, and compare +against independent canonical execution after a new batch is accepted. The +terminal-drained baseline itself need not equal canonical state at `C`. + +Automate this integration check where the artifacts are available, and require +passing evidence for the supported release before production deployment. Record +artifact versions, commands, checks, and timings; repeat when the state mapping, +checkpoint/recovery behavior, or relevant release artifacts change. A native +dump round-trip or a test using shared scheduler fixtures alone does not exercise +the canonical-machine export boundary. The +[Track 6 integration plan](../plans/2026-07-coordination-tracks.md#track-6--dump--application-api-redesign) +tracks the remaining tooling and validation work; this requirement is not an +implemented deployment gate. + +## Incident playbook + +The objective is a usable, independently trusted starting state. A newer sound +checkpoint reduces replay work; finding the exact first bad execution or the +latest possible sound checkpoint is not a prerequisite for recovery. + +1. **Stop and preserve.** Stop the sequencer and prevent automatic restarts. + Preserve its data directory, logs, and available archives before rebuilding. + Pause watchdog ticks while copying its selected checkpoint, manifest, + `head.json`, and configuration. Preserve affected client databases and their + checkpoint metadata separately. +2. **Establish the cause and reference.** Check deployment identity, canonical + machine image, bootstrap boundary, and the reported comparison boundary. + A configuration mismatch is different from faulty execution. Fix the cause + before running the replacement sequencer; retain an independently trusted + canonical machine or earlier checkpoint as the reference. +3. **Select a sound application checkpoint.** The watchdog's durable head is + its last successful comparison checkpoint, or its operator-trusted initial + checkpoint if no comparison succeeded. A failed comparison does not replace + it; initialization and idle ticks are not successful comparisons. Use that + canonical state, or independently validate a retained candidate at the same + exact L1 boundary. Current-state equality can establish a usable application + state without establishing that all earlier executions were correct. +4. **Prepare a restorable native bundle.** Validate the candidate's restored + application state, embedded count/clock, and next batch nonce against the + canonical reference at block `B`. Keep the artifact and its boundary metadata + together and record how its trust was established. The receipt alone is not + evidence that a faulty sequencer executed correctly. +5. **Rebuild in a fresh directory.** Use the invocation below. Recovery chooses + its post-flush stopping block `C`, replays from the trusted checkpoint, and + publishes a new era. Preserve that baseline artifact and its metadata for + client alignment before resumed operation can collect it. Resume independent + watchdog comparison when a new accepted comparison checkpoint is available. + +### Obtaining the recovery artifact + +The watchdog stores a whole CM, including scheduler state; it does not save a +native `/finalized_snapshot` archive. Use the application's rehearsed canonical +export command to obtain the native bundle, or a retained native archive whose +state and resume metadata can be validated against the canonical reference. +The mapping is required even when it is a direct extraction of a designated +drive. The generic watchdog does not implement that application-specific command. +A comparison file alone need not contain everything an engine requires to restore. + +Retain verified native archives outside sequencer GC if they are the intended +recovery source. The watchdog normally prunes its previous CM checkpoint, and +sequencer GC may remove the native artifact from the last passing comparison +after a newer batch is accepted. Downloading `/finalized_snapshot` after an +alarm can return the faulty newer state. The +[backup guide](../watchdog/operator-deployment.md#checkpoint-disk-usage-and-backups) +describes retention; the application runbook supplies the tested conversion. +Use its rehearsed earlier-checkpoint/genesis fallback when the preferred source +is unavailable. + +### Application-specific reader state + +A reader such as Bart's indexer owns more state than the sequencer application. +Choose its latest checkpoint whose execution provenance and indexing behavior +remain trustworthy after diagnosing the incident. Its boundary may differ from +the sequencer's chosen checkpoint. Compare against an independent reconstruction +of the required projection when needed; matching current balances or positions +does not validate historical transfers, deals, or portfolio records. + +Restore that complete client checkpoint and its scheduler continuation metadata, +then replay canonical L1 inputs through the replacement era's `C` and perform +the terminal drain. Establish agreement with the replacement application +baseline, including count `K`, before binding the reader to the new history +claim. If no client checkpoint can be trusted, rebuild its projection from a +trusted origin. A current application snapshot cannot supply omitted history. + +The [projection replay contract](../protocol/projection-replay.md) describes the +historical-input API, checkpoint preparation, and handoff metadata. Its reference +test exercises a trusted client checkpoint; the application-specific backup and +incident validation remain integration work. This manual procedure does not +require automated cross-era checkpoint matching. + ## Run a rebuild -Stop the old sequencer and resolve the cause of the failure. Choose a trusted -canonical application checkpoint; a recent one reduces replay work. Its state, -inclusion block, and next batch nonce must agree. After a bug, establish that -trust independently of the faulty local state, using a trusted canonical-machine -checkpoint or replay from a trusted origin. +Select and prepare the trusted checkpoint using the incident playbook above. The loader requires an application artifact, `info.toml`, and `checkpoint.toml`. The [recovery export workflow](../snapshots/lifecycle.md#http-and-recovery-exports) diff --git a/docs/watchdog/operator-deployment.md b/docs/watchdog/operator-deployment.md index f9b31d86..e7c3c7d4 100644 --- a/docs/watchdog/operator-deployment.md +++ b/docs/watchdog/operator-deployment.md @@ -381,6 +381,15 @@ checkpoint as that archive. The [snapshot backup workflow](../snapshots/lifecycle.md#http-and-recovery-exports) owns this separate restore path. +The [incident playbook](../recovery/cockroach.md#incident-playbook) covers +selecting a sound state, validating a native recovery bundle, and recovering +application-specific reader databases. It also identifies the CM-to-native +export and retention requirements that operator backups must cover. +Complete the application's +[recovery readiness procedure and drill](../recovery/cockroach.md#recovery-readiness-before-deployment) +before production deployment; a retained CM checkpoint is useful only with a +known, tested path back to a running native application. + ## Sequencer restart policy The sequencer's exit codes are the restart contract: 10 @@ -415,12 +424,13 @@ unclassified restart-with-backoff. Operational notes: also logs the latest row once at startup). Any death that did not return through the bracket — SIGKILL, OOM, a node reboot, a terminal runtime abort, a controller panic — leaves only the process logs. -- **Canonical divergence is the one manual path**: the sequencer freezes - the acceptance frontier - ([I15](../invariants.md#i15-divergence-marker-present--acceptance-frontier-frozen)), - refuses every command on that data directory, and the remedy is a - fresh-directory `setup --recovery` (cockroach). You will typically learn - of it from the watchdog before the sequencer tells you. +- **Untrustworthy local state requires manual recovery.** The reader's + content-identity divergence marker freezes the acceptance frontier and + blocks admission ([I15](../invariants.md#i15-divergence-marker-present--acceptance-frontier-frozen)). + A watchdog state mismatch independently signals application-state + disagreement. Diagnose the cause and follow the + [incident playbook](../recovery/cockroach.md#incident-playbook) for a + fresh-directory rebuild when needed; standard recovery does not repair bugs. ## Troubleshooting (live deployments) diff --git a/sdk/rust-client/src/errors.rs b/sdk/rust-client/src/errors.rs index ec35b321..cd7ed739 100644 --- a/sdk/rust-client/src/errors.rs +++ b/sdk/rust-client/src/errors.rs @@ -84,3 +84,15 @@ pub enum SnapshotError { #[error("invalid snapshot metadata: {0}")] Metadata(String), } + +#[derive(Debug, Error)] +pub enum HistoryReadError { + #[error("history request failed: {0}")] + Request(#[from] reqwest::Error), + #[error(transparent)] + History(#[from] sequencer_core::history::HistoryPolicyError), + #[error("history request rejected with status {status}: {body}")] + Http { status: u16, body: String }, + #[error("invalid history response: {0}")] + Decode(#[from] serde_json::Error), +} diff --git a/sdk/rust-client/src/history.rs b/sdk/rust-client/src/history.rs new file mode 100644 index 00000000..35a5da2e --- /dev/null +++ b/sdk/rust-client/src/history.rs @@ -0,0 +1,359 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +use crate::{ + EraId, HistoricalL1InputStart, HistoricalL1InputsPage, HistoryInfo, HistoryPolicyError, + HistoryReadError, RecoveryGeneration, SequencerClient, +}; + +impl SequencerClient { + /// Discover history, optionally requiring the era selected for bootstrap. + /// A compatibility query supplies both the checkpoint's era and its generation. + pub async fn history( + &self, + expected_era: Option, + from_generation: Option, + ) -> Result { + let mut query: Vec<_> = expected_era + .map(|era| ("era_id", era.to_string())) + .into_iter() + .collect(); + if let Some(generation) = from_generation { + query.push(("from_generation", generation.get().to_string())); + } + let request = self + .http_client + .get(format!("{}/history", self.endpoint.trim_end_matches('/'))) + .query(&query) + .timeout(self.request_timeout); + let body = read_history_response(request).await?; + Ok(serde_json::from_str(&body)?) + } + + /// Read one complete raw-input page under the configured request deadline. + /// Continue with its `next_input_index`; a short page is not necessarily EOF. + /// Use `with_request_timeout` when historical transfers need a longer deadline. + pub async fn historical_l1_inputs( + &self, + era: EraId, + start: HistoricalL1InputStart, + limit: Option, + ) -> Result { + let mut query = vec![("era_id", era.to_string())]; + query.push(match start { + HistoricalL1InputStart::NextInputIndex(index) => { + ("next_input_index", index.to_string()) + } + HistoricalL1InputStart::AfterBlock(block) => ("after_block", block.to_string()), + }); + if let Some(limit) = limit { + query.push(("limit", limit.to_string())); + } + let request = self + .http_client + .get(format!( + "{}/historical-l1-inputs", + self.endpoint.trim_end_matches('/') + )) + .query(&query) + .timeout(self.request_timeout); + let body = read_history_response(request).await?; + Ok(serde_json::from_str(&body)?) + } +} + +async fn read_history_response( + request: reqwest::RequestBuilder, +) -> Result { + let response = request.send().await?; + let status = response.status().as_u16(); + let body = response.text().await?; + if status == 409 + && let Ok(policy) = serde_json::from_str::(&body) + { + return Err(HistoryReadError::History(policy)); + } + if status != 200 { + return Err(HistoryReadError::Http { status, body }); + } + Ok(body) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::time::Duration; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + use tokio::net::{TcpListener, TcpStream}; + use tokio::task::JoinHandle; + + const ERA: &str = "00112233-4455-4677-8899-aabbccddeeff"; + + async fn request_headers(stream: &mut TcpStream) -> String { + let mut headers = Vec::new(); + while !headers.ends_with(b"\r\n\r\n") { + let mut byte = [0]; + assert_eq!(stream.read(&mut byte).await.unwrap(), 1); + headers.push(byte[0]); + assert!(headers.len() <= 8192, "request headers exceed test bound"); + } + String::from_utf8(headers).unwrap() + } + + async fn serve_once(status: u16, body: String) -> (SequencerClient, JoinHandle) { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let endpoint = format!("http://{}/", listener.local_addr().unwrap()); + let server = tokio::spawn(async move { + let (mut stream, _) = listener.accept().await.unwrap(); + let headers = request_headers(&mut stream).await; + stream + .write_all( + format!( + "HTTP/1.1 {status} Test\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", + body.len() + ) + .as_bytes(), + ) + .await + .unwrap(); + headers + }); + (SequencerClient::new(endpoint).unwrap(), server) + } + + fn request_url(headers: &str) -> reqwest::Url { + let target = headers + .lines() + .next() + .unwrap() + .split_whitespace() + .nth(1) + .unwrap(); + reqwest::Url::parse(&format!("http://localhost{target}")).unwrap() + } + + fn history_body() -> String { + serde_json::json!({ + "deployment": { + "chain_id": 31337, + "app_address": "0x1111111111111111111111111111111111111111", + "input_box_address": "0x2222222222222222222222222222222222222222", + "app_deployment_block": 1, + "batch_submitter_address": "0x3333333333333333333333333333333333333333" + }, + "history": { + "version": { "era_id": ERA, "recovery_generation": 2 }, + "available_from": 7, + "head": 12 + }, + "baseline": { + "l1_stop_block": 1240, + "l1_end_input_index": 8, + "next_batch_nonce": 2 + }, + "accepted_checkpoint": { + "inclusion_block": 1250, + "executed_input_count": 10, + "next_batch_nonce": 3 + }, + "compatibility": null + }) + .to_string() + } + + fn page_body() -> String { + serde_json::json!({ + "era_id": ERA, + "l1_stop_block": 1240, + "end_input_index": 8, + "next_input_index": 6, + "items": [{ + "input_index": 5, + "sender": "0x3333333333333333333333333333333333333333", + "payload": "0x00ff80", + "block_number": 1230, + "block_timestamp": 1700014760_u64, + "transaction_hash": "0x4444444444444444444444444444444444444444444444444444444444444444" + }] + }) + .to_string() + } + + #[tokio::test] + async fn discovery_decodes_history_and_encodes_optional_era() { + for expected_era in [None, Some(ERA.parse().unwrap())] { + let (client, server) = serve_once(200, history_body()).await; + let info = client.history(expected_era, None).await.unwrap(); + assert_eq!(info.history.available_from.get(), 7); + assert_eq!(info.history.head.get(), 12); + assert_eq!(info.history.version.era_id.to_string(), ERA); + assert_eq!(info.baseline.l1_stop_block, 1240); + assert!(info.compatibility.is_none()); + let url = request_url(&server.await.unwrap()); + assert_eq!(url.path(), "/history"); + let query: Vec<_> = url.query_pairs().collect(); + if expected_era.is_some() { + assert_eq!(query, [("era_id".into(), ERA.into())]); + } else { + assert!(query.is_empty()); + } + } + } + + #[tokio::test] + async fn compatibility_encodes_checkpoint_generation_and_decodes_preserved_boundary() { + let mut body: serde_json::Value = serde_json::from_str(&history_body()).unwrap(); + body["compatibility"] = serde_json::json!({ + "from_generation": 1, + "preserved_input_count": 9 + }); + let (client, server) = serve_once(200, body.to_string()).await; + let info = client + .history(Some(ERA.parse().unwrap()), Some(RecoveryGeneration::new(1))) + .await + .unwrap(); + let compatibility = info.compatibility.unwrap(); + assert_eq!(compatibility.from_generation.get(), 1); + assert_eq!(compatibility.preserved_input_count.get(), 9); + assert_eq!(info.history.version.recovery_generation.get(), 2); + let url = request_url(&server.await.unwrap()); + assert_eq!(url.path(), "/history"); + assert_eq!( + url.query_pairs().collect::>(), + [ + ("era_id".into(), ERA.into()), + ("from_generation".into(), "1".into()) + ] + ); + } + + #[tokio::test] + async fn pages_encode_one_selector_and_decode_original_binary_fields() { + for (start, limit, selector, value) in [ + ( + HistoricalL1InputStart::NextInputIndex(5), + Some(1), + "next_input_index", + "5", + ), + ( + HistoricalL1InputStart::AfterBlock(10), + None, + "after_block", + "10", + ), + ] { + let (client, server) = serve_once(200, page_body()).await; + let page = client + .historical_l1_inputs(ERA.parse().unwrap(), start, limit) + .await + .unwrap(); + assert_eq!(page.next_input_index, 6); + assert_eq!(page.items[0].input_index, 5); + assert_eq!(page.items[0].payload.as_ref(), &[0x00, 0xff, 0x80]); + assert_eq!(page.items[0].sender.as_slice(), &[0x33; 20]); + assert_eq!(page.items[0].transaction_hash.as_slice(), &[0x44; 32]); + let url = request_url(&server.await.unwrap()); + assert_eq!(url.path(), "/historical-l1-inputs"); + let query: std::collections::BTreeMap<_, _> = url.query_pairs().collect(); + assert_eq!(query.get("era_id").unwrap(), ERA); + assert_eq!(query.get(selector).unwrap(), value); + assert_eq!(query.len(), if limit.is_some() { 3 } else { 2 }); + if let Some(limit) = limit { + assert_eq!(query.get("limit").unwrap(), &limit.to_string()); + } + } + } + + #[tokio::test] + async fn era_change_is_typed_for_both_reads() { + let policy = HistoryPolicyError::EraChanged { + current: crate::HistoryVersion { + era_id: "11111111-1111-4111-8111-111111111111".parse().unwrap(), + recovery_generation: sequencer_core::history::RecoveryGeneration::new(0), + }, + }; + for page_request in [false, true] { + let (client, server) = serve_once(409, serde_json::to_string(&policy).unwrap()).await; + let result = if page_request { + client + .historical_l1_inputs( + ERA.parse().unwrap(), + HistoricalL1InputStart::NextInputIndex(0), + None, + ) + .await + .map(|_| ()) + } else { + client + .history(Some(ERA.parse().unwrap()), None) + .await + .map(|_| ()) + }; + assert!(matches!(result, Err(HistoryReadError::History(actual)) if actual == policy)); + server.await.unwrap(); + } + } + + #[tokio::test] + async fn other_refusals_preserve_status_and_body() { + for status in [400, 409, 429, 503] { + let body = format!("refusal {status}"); + let (client, server) = serve_once(status, body.clone()).await; + assert!(matches!( + client.history(None, None).await, + Err(HistoryReadError::Http { status: actual_status, body: actual_body }) + if actual_status == status && actual_body == body + )); + server.await.unwrap(); + } + } + + #[tokio::test] + async fn malformed_success_is_a_decode_error() { + let (client, server) = serve_once(200, "{}".into()).await; + assert!(matches!( + client.history(None, None).await, + Err(HistoryReadError::Decode(_)) + )); + server.await.unwrap(); + } + + #[tokio::test] + async fn history_deadline_covers_the_response_body() { + for page_request in [false, true] { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let endpoint = format!("http://{}", listener.local_addr().unwrap()); + let server = tokio::spawn(async move { + let (mut stream, _) = listener.accept().await.unwrap(); + request_headers(&mut stream).await; + stream + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 100\r\n\r\n{") + .await + .unwrap(); + std::future::pending::<()>().await; + }); + let client = SequencerClient::new(endpoint) + .unwrap() + .with_request_timeout(Duration::from_millis(100)); + let result = tokio::time::timeout(Duration::from_secs(5), async { + if page_request { + client + .historical_l1_inputs( + ERA.parse().unwrap(), + HistoricalL1InputStart::NextInputIndex(0), + None, + ) + .await + .map(|_| ()) + } else { + client.history(None, None).await.map(|_| ()) + } + }) + .await + .expect("the configured deadline must include body transfer"); + server.abort(); + assert!(matches!(result, Err(HistoryReadError::Request(error)) if error.is_timeout())); + } + } +} diff --git a/sdk/rust-client/src/lib.rs b/sdk/rust-client/src/lib.rs index 7003c5b0..79302ecd 100644 --- a/sdk/rust-client/src/lib.rs +++ b/sdk/rust-client/src/lib.rs @@ -2,13 +2,20 @@ // SPDX-License-Identifier: Apache-2.0 (see LICENSE) mod errors; +mod history; pub use errors::{ - ClientBuildError, GetFeeError, SnapshotError, SubmitRejected, SubmitTxError, SubscribeError, + ClientBuildError, GetFeeError, HistoryReadError, SnapshotError, SubmitRejected, SubmitTxError, + SubscribeError, }; pub use sequencer_core::history::{ - ExecutedInputCount, HistoryClaim, HistoryPolicyError, HistoryVersion, + EraId, ExecutedInputCount, HistoryBounds, HistoryClaim, HistoryPolicyError, HistoryVersion, + RecoveryGeneration, +}; +pub use sequencer_core::history_api::{ + AcceptedCheckpoint, HistoricalL1Input, HistoricalL1InputStart, HistoricalL1InputsPage, + HistoryBaseline, HistoryCompatibility, HistoryDeployment, HistoryInfo, }; use sequencer_core::api::{FeeResponse, TxRequest, TxResponse}; diff --git a/sequencer-core/src/history_api.rs b/sequencer-core/src/history_api.rs new file mode 100644 index 00000000..d8312fb4 --- /dev/null +++ b/sequencer-core/src/history_api.rs @@ -0,0 +1,81 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +//! Historical scheduler inputs and their handoff to the application feed. + +use alloy_primitives::{Address, B256, Bytes}; +use serde::{Deserialize, Serialize}; + +use crate::history::{EraId, ExecutedInputCount, HistoryBounds, RecoveryGeneration}; + +pub const HISTORICAL_INPUT_MAX_ITEMS: usize = 256; +/// A larger first input is served alone, preserving progress through any history. +pub const HISTORICAL_INPUT_PAYLOAD_TARGET_BYTES: usize = 1024 * 1024; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct HistoryDeployment { + pub chain_id: u64, + pub app_address: Address, + pub input_box_address: Address, + pub app_deployment_block: u64, + pub batch_submitter_address: Address, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub struct HistoryBaseline { + pub l1_stop_block: u64, + /// Exclusive end of the per-application InputBox prefix through the stop block. + pub l1_end_input_index: u64, + pub next_batch_nonce: u64, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub struct AcceptedCheckpoint { + pub inclusion_block: u64, + pub executed_input_count: ExecutedInputCount, + pub next_batch_nonce: u64, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct HistoryInfo { + pub deployment: HistoryDeployment, + pub history: HistoryBounds, + pub baseline: HistoryBaseline, + /// The accepted boundary shares `history.version`; it does not certify client state. + pub accepted_checkpoint: Option, + pub compatibility: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub struct HistoryCompatibility { + pub from_generation: RecoveryGeneration, + /// Checkpoint counts up to and including this boundary preserve their input prefix. + pub preserved_input_count: ExecutedInputCount, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HistoricalL1InputStart { + NextInputIndex(u64), + AfterBlock(u64), +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct HistoricalL1Input { + pub input_index: u64, + pub sender: Address, + /// Original inner application/batch payload, including malformed batches. + pub payload: Bytes, + pub block_number: u64, + pub block_timestamp: u64, + pub transaction_hash: B256, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct HistoricalL1InputsPage { + pub era_id: EraId, + pub l1_stop_block: u64, + pub end_input_index: u64, + /// Only equality with `end_input_index` establishes completion, not page length. + pub next_input_index: u64, + pub items: Vec, +} diff --git a/sequencer-core/src/lib.rs b/sequencer-core/src/lib.rs index 7c521333..d8ab1f3c 100644 --- a/sequencer-core/src/lib.rs +++ b/sequencer-core/src/lib.rs @@ -10,6 +10,7 @@ pub mod batch; pub mod broadcast; pub mod fee; pub mod history; +pub mod history_api; pub mod l2_tx; pub mod protocol; pub mod scheduler; diff --git a/sequencer/src/egress/api/history.rs b/sequencer/src/egress/api/history.rs new file mode 100644 index 00000000..e14cd260 --- /dev/null +++ b/sequencer/src/egress/api/history.rs @@ -0,0 +1,276 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +//! Coherent history metadata and bounded pages of the immutable era L1 prefix. + +use std::io::Cursor; +use std::pin::Pin; +use std::sync::Arc; +use std::task::{Context, Poll}; + +use axum::body::Body; +use axum::extract::rejection::QueryRejection; +use axum::extract::{Query, State}; +use axum::http::{StatusCode, header}; +use axum::response::{IntoResponse, Response}; +use axum::routing::get; +use axum::{Json, Router}; +use sequencer_core::history::{EraId, RecoveryGeneration}; +use sequencer_core::history_api::{HISTORICAL_INPUT_MAX_ITEMS, HistoricalL1InputStart}; +use serde::Deserialize; +use tokio::io::{AsyncRead, ReadBuf}; +use tokio::sync::{OwnedSemaphorePermit, Semaphore}; +use tokio_util::io::ReaderStream; + +use crate::http::{ApiError, StorageTaskError, storage_task}; +use crate::runtime::shutdown::RuntimeScope; +use crate::storage::{HistoricalReadError, Storage}; + +const MAX_HISTORICAL_RESPONSES: usize = 8; + +struct HistoryState { + db_path: String, + shutdown: RuntimeScope, + responses: Arc, +} + +pub(super) fn router(db_path: String, shutdown: RuntimeScope) -> Router { + Router::new() + .route("/history", get(history)) + .route("/historical-l1-inputs", get(historical_l1_inputs)) + .with_state(Arc::new(HistoryState { + db_path, + shutdown, + responses: Arc::new(Semaphore::new(MAX_HISTORICAL_RESPONSES)), + })) +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct HistoryQuery { + era_id: Option, + from_generation: Option, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct HistoricalInputsQuery { + era_id: EraId, + next_input_index: Option, + after_block: Option, + limit: Option, +} + +impl HistoricalInputsQuery { + fn start(&self) -> Result { + match (self.next_input_index, self.after_block) { + (Some(index), None) => Ok(HistoricalL1InputStart::NextInputIndex(index)), + (None, Some(block)) => Ok(HistoricalL1InputStart::AfterBlock(block)), + _ => Err(ApiError::bad_request( + "provide exactly one of next_input_index or after_block", + )), + } + } +} + +async fn history( + State(state): State>, + query: Result, QueryRejection>, +) -> Response { + let Query(query) = match query { + Ok(query) => query, + Err(error) => return ApiError::bad_request(error.body_text()).into_response(), + }; + if query.from_generation.is_some() && query.era_id.is_none() { + return ApiError::bad_request("from_generation requires era_id").into_response(); + } + if state.shutdown.is_shutdown_requested() { + return ApiError::unavailable("sequencer shutting down").into_response(); + } + let db_path = state.db_path.clone(); + match storage_task(state.shutdown.clone(), "read history metadata", move |_| { + Ok(Storage::open_read_only(&db_path)?.history_info(query.era_id, query.from_generation)?) + }) + .await + { + Ok(info) => Json(info).into_response(), + Err(error) => read_error(error), + } +} + +async fn historical_l1_inputs( + State(state): State>, + query: Result, QueryRejection>, +) -> Response { + let Query(query) = match query { + Ok(query) => query, + Err(error) => return ApiError::bad_request(error.body_text()).into_response(), + }; + let start = match query.start() { + Ok(start) => start, + Err(error) => return error.into_response(), + }; + if state.shutdown.is_shutdown_requested() { + return ApiError::unavailable("sequencer shutting down").into_response(); + } + let permit = match state.responses.clone().try_acquire_owned() { + Ok(permit) => permit, + Err(_) => return ApiError::overloaded("historical response limit reached").into_response(), + }; + let db_path = state.db_path.clone(); + let result = storage_task( + state.shutdown.clone(), + "read historical L1 inputs", + move |_| { + // The blocking task owns admission even if its HTTP request is cancelled. + let page = Storage::open_read_only(&db_path)?.historical_l1_inputs( + query.era_id, + start, + query.limit.unwrap_or(HISTORICAL_INPUT_MAX_ITEMS), + )?; + let bytes = serde_json::to_vec(&page)?; + Ok((bytes, permit)) + }, + ) + .await; + match result { + Ok((bytes, permit)) => ( + [(header::CONTENT_TYPE, "application/json")], + Body::from_stream(ReaderStream::new(HistoricalResponseBody { + bytes: Cursor::new(bytes), + _permit: permit, + })), + ) + .into_response(), + Err(error) => read_error(error), + } +} + +fn read_error(error: StorageTaskError) -> Response { + match error.downcast_ref::() { + Some(HistoricalReadError::Policy(policy)) => { + return (StatusCode::CONFLICT, Json(*policy)).into_response(); + } + Some(HistoricalReadError::BadRequest(message)) => { + return ApiError::bad_request(message.clone()).into_response(); + } + _ => {} + } + tracing::warn!(%error, "history read unavailable"); + ApiError::unavailable("history read unavailable").into_response() +} + +struct HistoricalResponseBody { + bytes: Cursor>, + _permit: OwnedSemaphorePermit, +} + +impl AsyncRead for HistoricalResponseBody { + fn poll_read( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &mut ReadBuf<'_>, + ) -> Poll> { + Pin::new(&mut self.bytes).poll_read(cx, buf) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const ERA: &str = "11111111-1111-4111-8111-111111111111"; + + fn state() -> Arc { + Arc::new(HistoryState { + db_path: "unused: invalid queries do not access storage".to_owned(), + shutdown: RuntimeScope::default(), + responses: Arc::new(Semaphore::new(1)), + }) + } + + #[tokio::test] + async fn malformed_queries_return_bad_request_json_before_storage() { + for query in [ + "", + "?era_id=bad&next_input_index=0", + &format!("?era_id={ERA}"), + &format!("?era_id={ERA}&next_input_index=0&after_block=0"), + &format!("?era_id={ERA}&next_input_index=-1"), + &format!("?era_id={ERA}&next_input_index=0&unrecognized=1"), + ] { + let uri = format!("/historical-l1-inputs{query}").parse().unwrap(); + let response = historical_l1_inputs(State(state()), Query::try_from_uri(&uri)).await; + assert_eq!(response.status(), StatusCode::BAD_REQUEST, "{query}"); + let bytes = axum::body::to_bytes(response.into_body(), 4096) + .await + .unwrap(); + let error: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + assert_eq!(error["code"], "BAD_REQUEST"); + } + for query in [ + "?from_generation=0", + &format!("?era_id={ERA}&from_generation=-1"), + &format!("?era_id={ERA}&from_generation=invalid"), + &format!("?era_id={ERA}&unrecognized=1"), + ] { + let uri = format!("/history{query}").parse().unwrap(); + let response = history(State(state()), Query::try_from_uri(&uri)).await; + assert_eq!(response.status(), StatusCode::BAD_REQUEST, "{query}"); + let bytes = axum::body::to_bytes(response.into_body(), 4096) + .await + .unwrap(); + let error: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + assert_eq!(error["code"], "BAD_REQUEST"); + } + } + + #[tokio::test] + async fn shutdown_and_overload_refuse_before_opening_storage() { + let state = state(); + let uri = format!("/historical-l1-inputs?era_id={ERA}&next_input_index=0") + .parse() + .unwrap(); + let permit = state.responses.clone().try_acquire_owned().unwrap(); + let response = historical_l1_inputs(State(state.clone()), Query::try_from_uri(&uri)).await; + assert_eq!(response.status(), StatusCode::TOO_MANY_REQUESTS); + drop(permit); + state.shutdown.request_shutdown(); + let response = historical_l1_inputs(State(state.clone()), Query::try_from_uri(&uri)).await; + assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); + for query in [ + "".to_owned(), + format!("?era_id={ERA}"), + format!("?era_id={ERA}&from_generation=0"), + ] { + let uri = format!("/history{query}").parse().unwrap(); + let response = history(State(state.clone()), Query::try_from_uri(&uri)).await; + assert_eq!( + response.status(), + StatusCode::SERVICE_UNAVAILABLE, + "{query}" + ); + } + } + + #[tokio::test] + async fn response_body_holds_admission_until_consumed_or_dropped() { + let permits = Arc::new(Semaphore::new(1)); + for consume in [false, true] { + let body = Body::from_stream(ReaderStream::new(HistoricalResponseBody { + bytes: Cursor::new(vec![1; 100_000]), + _permit: permits.clone().try_acquire_owned().unwrap(), + })); + assert!(permits.clone().try_acquire_owned().is_err()); + if consume { + assert_eq!( + axum::body::to_bytes(body, 100_000).await.unwrap().len(), + 100_000 + ); + } else { + drop(body); + } + assert_eq!(permits.available_permits(), 1); + } + } +} diff --git a/sequencer/src/egress/api/mod.rs b/sequencer/src/egress/api/mod.rs index 39a7db9c..37e2d643 100644 --- a/sequencer/src/egress/api/mod.rs +++ b/sequencer/src/egress/api/mod.rs @@ -1,10 +1,10 @@ // (c) Cartesi and individual authors (see AUTHORS) // SPDX-License-Identifier: Apache-2.0 (see LICENSE) -//! Egress HTTP API routes: WebSocket subscribe + k8s-style health probes. -//! Additional read endpoints will land here. +//! Internal history, snapshot, WebSocket, and health endpoints. mod health; +mod history; mod snapshot; mod state; mod subscribe; @@ -31,6 +31,7 @@ pub(crate) fn router( shutdown: RuntimeScope, snapshot_release_scheduler: ReleaseScheduler, ) -> Router { + let history_router = history::router(snapshot_state.db_path.clone(), shutdown.clone()); let subscribe_router = Router::new() .route("/ws/subscribe", get(subscribe::subscribe_l2_txs)) .with_state(subscribe_state); @@ -43,6 +44,7 @@ pub(crate) fn router( subscribe_router .merge(health_router) + .merge(history_router) .merge(snapshot::router( snapshot_state, shutdown, diff --git a/sequencer/src/integration_tests/historical_bootstrap.rs b/sequencer/src/integration_tests/historical_bootstrap.rs new file mode 100644 index 00000000..bfffbd44 --- /dev/null +++ b/sequencer/src/integration_tests/historical_bootstrap.rs @@ -0,0 +1,553 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +//! A reader restores its own transfer history, replays raw L1 through the +//! canonical scheduler, and crosses the rebuilt baseline into the live feed. + +mod recovery_compatibility; + +use std::net::SocketAddr; +use std::time::Duration; + +use alloy::signers::{SignerSync, local::PrivateKeySigner}; +use alloy_primitives::{Address, B256, U256}; +use alloy_sol_types::{Eip712Domain, SolCall, SolStruct}; +use app_core::application::{ + DepositNotice, Method, Transfer, TransferNotice, WalletApp, WalletConfig, +}; +use futures_util::StreamExt; +use sequencer_core::api::WsTxMessage; +use sequencer_core::application::{ + AppOutput, AppOutputs, Application, CanonicalState, execute_direct_input, +}; +use sequencer_core::batch::{Batch, Frame, WireUserOp}; +use sequencer_core::history::{ExecutedInputCount, HistoryClaim, HistoryPolicyError}; +use sequencer_core::history_api::{AcceptedCheckpoint, HistoricalL1InputStart}; +use sequencer_core::l2_tx::DirectInput; +use sequencer_core::scheduler::{ + BatchRejectReason, ProcessOutcome, Scheduler, SchedulerConfig, SchedulerInput, +}; +use sequencer_core::user_op::UserOp; +use sequencer_rust_client::{HistoryReadError, SequencerClient}; +use ssz::Encode; +use tokio::sync::mpsc; +use tokio_tungstenite::tungstenite::Message; + +use crate::egress::l2_tx_feed::{L2TxFeed, L2TxFeedConfig}; +use crate::http::{self, ApiConfig}; +use crate::ingress::inclusion_lane::{PendingUserOp, dump_info}; +use crate::runtime::shutdown::RuntimeScope; +use crate::storage::test_helpers::{default_protocol_timing, pin_test_deployment_identity}; +use crate::storage::{ + DirectInputExecution, FrontierMode, IngestedSafeInput, LifecycleCommand, SafeInputRange, + Storage, +}; + +use super::common::temp_db; + +const SUBMITTER: Address = Address::repeat_byte(0x33); +const APP: Address = Address::repeat_byte(0x11); +const RECIPIENT: Address = Address::repeat_byte(0x44); +const STOP: u64 = 1240; + +fn domain() -> Eip712Domain { + sequencer_core::build_input_domain(1, APP) +} + +fn config() -> WalletConfig { + WalletConfig { + sequencer_address: SUBMITTER, + ..WalletConfig::default() + } +} + +fn deposit(block: u64, recipient: Address, amount: u64) -> IngestedSafeInput { + let mut payload = Vec::new(); + payload.extend_from_slice(config().supported_erc20_token.as_slice()); + payload.extend_from_slice(recipient.as_slice()); + payload.extend_from_slice(&U256::from(amount).to_be_bytes::<32>()); + raw(config().erc20_portal_address, block, payload) +} + +fn raw(sender: Address, block: u64, payload: Vec) -> IngestedSafeInput { + IngestedSafeInput { + sender, + payload, + block_number: block, + block_timestamp: 1_700_000_000 + block * 12, + transaction_hash: B256::repeat_byte(u8::try_from(block % 251).unwrap()), + } +} + +fn transfer_batch( + signer: &PrivateKeySigner, + nonce: u32, + block: u64, + safe_block: u64, + amount: u64, +) -> IngestedSafeInput { + let op = UserOp { + nonce, + max_fee: 0, + data: Method::Transfer(Transfer { + amount: U256::from(amount), + to: RECIPIENT, + }) + .as_ssz_bytes() + .into(), + }; + let signature = signer + .sign_hash_sync(&op.eip712_signing_hash(&domain())) + .unwrap(); + raw( + SUBMITTER, + block, + Batch { + nonce: u64::from(nonce), + frames: vec![Frame { + safe_block, + fee_price: 0, + user_ops: vec![WireUserOp { + nonce, + max_fee: op.max_fee, + data: op.data.to_vec(), + signature: signature.as_bytes().to_vec(), + }], + }], + } + .as_ssz_bytes(), + ) +} + +fn notices(outputs: AppOutputs) -> Vec> { + outputs + .into_iter() + .map(|output| match output { + AppOutput::Notice(bytes) => bytes, + other => panic!("unexpected output in transfer-history fixture: {other:?}"), + }) + .collect() +} + +fn execute( + scheduler: &mut Scheduler, + input: &IngestedSafeInput, +) -> sequencer_core::scheduler::ProcessResult { + scheduler + .process_input(SchedulerInput { + sender: input.sender, + inclusion_block: input.block_number, + domain: domain(), + payload: input.payload.clone(), + }) + .unwrap() +} + +struct Server { + addr: SocketAddr, + shutdown: RuntimeScope, + task: Option, + _rx: mpsc::Receiver, +} + +impl Drop for Server { + fn drop(&mut self) { + self.shutdown.request_shutdown(); + if let Some(task) = self.task.take() { + task.abort(); + } + } +} + +impl Server { + async fn start(db_path: &str) -> Self { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let shutdown = RuntimeScope::default(); + let (tx, rx) = mpsc::channel(1); + let feed = L2TxFeed::new( + db_path.to_owned(), + shutdown.clone(), + L2TxFeedConfig::default(), + ); + let task = http::start_on_listener( + listener, + tx, + shutdown.clone(), + feed, + ApiConfig::new(domain(), WalletApp::max_method_payload_bytes()), + http::SnapshotState { + db_path: db_path.to_owned(), + state_file_in_dump: |prefix| { + WalletApp::state_file_in_dump(&dump_info::app_prefix(prefix)) + }, + }, + ); + Self { + addr, + shutdown, + task: Some(task), + _rx: rx, + } + } + + async fn stop(mut self) { + self.shutdown.request_shutdown(); + tokio::time::timeout(Duration::from_secs(3), self.task.take().unwrap()) + .await + .unwrap() + .unwrap() + .unwrap(); + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn historical_bootstrap_restores_transfer_history_and_hands_off_at_baseline() { + let signer: PrivateKeySigner = format!("{:064x}", 1).parse().unwrap(); + let user = signer.address(); + let inputs = vec![ + deposit(5, user, 100), + deposit(12, user, 20), + transfer_batch(&signer, 0, 20, 10, 7), + deposit(20, user, 30), + deposit(24, user, 40), + raw(SUBMITTER, 1230, vec![0]), + transfer_batch(&signer, 1, 1232, 1228, 11), + deposit(1235, user, 50), + ]; + + let mut reference = Scheduler::new(WalletApp::new(config()), SchedulerConfig::new(SUBMITTER)); + let mut expected_history = Vec::new(); + for input in &inputs { + expected_history.extend(notices(execute(&mut reference, input).outputs)); + } + expected_history.extend(notices(reference.drain_covered_at(STOP).unwrap())); + let (mut reference_app, reference_nonce) = reference.finish(); + assert_eq!(reference_app.executed_input_count().get(), 7); + assert_eq!(reference_app.last_executed_safe_block(), 1235); + assert_eq!(reference_nonce, 2); + let deposit_notice = |amount| { + DepositNotice { + token: config().supported_erc20_token, + sender: user, + amount: U256::from(amount), + } + .abi_encode() + }; + let transfer_notice = |amount| { + TransferNotice { + sender: user, + recipient: RECIPIENT, + amount: U256::from(amount), + } + .abi_encode() + }; + assert_eq!( + expected_history, + vec![ + deposit_notice(100_u64), + transfer_notice(7_u64), + deposit_notice(20), + deposit_notice(30), + deposit_notice(40), + transfer_notice(11), + deposit_notice(50) + ] + ); + + let db = temp_db("historical-projection-bootstrap"); + let dumps = tempfile::tempdir().unwrap(); + let baseline_dump = dumps.path().join("baseline"); + dump_info::create_dump_dir_with_info( + &mut reference_app, + &baseline_dump, + &dump_info::DumpInfo { + format_version: dump_info::FORMAT_VERSION, + next_batch_nonce: reference_nonce, + }, + ) + .unwrap(); + let mut storage = Storage::initialize_for_command(&db.path, LifecycleCommand::Rebuild).unwrap(); + pin_test_deployment_identity(&mut storage, SUBMITTER); + storage + .append_ingested_safe_inputs_with_timestamp( + STOP, + 1_700_000_000 + STOP * 12, + &inputs, + SUBMITTER, + &default_protocol_timing(), + FrontierMode::DeferUntilAnchorSet, + ) + .unwrap(); + storage + .complete_baseline_setup( + &baseline_dump, + reference_app.executed_input_count(), + STOP, + reference_nonce, + true, + ) + .unwrap(); + let server = Server::start(&db.path).await; + let client = SequencerClient::new(format!("http://{}", server.addr)).unwrap(); + let metadata = client.history(None, None).await.unwrap(); + let era = metadata.history.version.era_id; + assert_eq!(metadata.history.available_from.get(), 7); + assert_eq!(metadata.history.head.get(), 7); + assert_eq!(metadata.baseline.l1_stop_block, STOP); + assert_eq!(metadata.baseline.l1_end_input_index, 8); + assert_eq!(metadata.baseline.next_batch_nonce, reference_nonce); + assert_eq!(metadata.accepted_checkpoint, None); + assert_eq!(metadata.deployment.batch_submitter_address, SUBMITTER); + assert_eq!( + sequencer_core::build_input_domain( + metadata.deployment.chain_id, + metadata.deployment.app_address + ), + domain() + ); + let mut other_era_bytes = *era.as_bytes(); + other_era_bytes[0] ^= 1; + let other_era = sequencer_core::history::EraId::from_bytes(other_era_bytes).unwrap(); + assert!(matches!( + client.history(Some(other_era), None).await, + Err(HistoryReadError::History(HistoryPolicyError::EraChanged { current })) + if current == metadata.history.version + )); + assert!(matches!( + client + .historical_l1_inputs( + other_era, + HistoricalL1InputStart::NextInputIndex(u64::MAX), + None + ) + .await, + Err(HistoryReadError::History( + HistoryPolicyError::EraChanged { .. } + )) + )); + assert!(matches!( + client + .historical_l1_inputs(era, HistoricalL1InputStart::NextInputIndex(9), None) + .await, + Err(HistoryReadError::Http { status: 400, .. }) + )); + + // The reader's saved projection includes notices that the core wallet dump + // cannot reconstruct. Prepare a trusted end-of-block B=20 checkpoint. + let mut checkpoint_scheduler = + Scheduler::new(WalletApp::new(config()), SchedulerConfig::new(SUBMITTER)); + let mut saved_history = Vec::new(); + for input in &inputs[..4] { + saved_history.extend(notices(execute(&mut checkpoint_scheduler, input).outputs)); + } + assert_eq!(checkpoint_scheduler.queued_direct_len(), 2); + let (mut checkpoint_app, next_batch_nonce) = checkpoint_scheduler.finish(); + let receipt = AcceptedCheckpoint { + inclusion_block: 20, + executed_input_count: checkpoint_app.executed_input_count(), + next_batch_nonce, + }; + assert_eq!(receipt.executed_input_count.get(), 2); + assert_eq!(checkpoint_app.last_executed_safe_block(), 10); + let backup = dumps.path().join("reader-checkpoint"); + std::fs::create_dir(&backup).unwrap(); + checkpoint_app.create_dump(&backup.join("wallet")).unwrap(); + std::fs::write( + backup.join("projection.json"), + serde_json::to_vec(&(receipt, &saved_history)).unwrap(), + ) + .unwrap(); + drop(checkpoint_app); + drop(saved_history); + + let restored_app = WalletApp::from_dump(&backup.join("wallet")).unwrap(); + let (receipt, mut history): (AcceptedCheckpoint, Vec>) = + serde_json::from_slice(&std::fs::read(backup.join("projection.json")).unwrap()).unwrap(); + assert_eq!( + restored_app.executed_input_count(), + receipt.executed_input_count + ); + let mut start = HistoricalL1InputStart::AfterBlock(restored_app.last_executed_safe_block()); + let mut replay = Scheduler::resume_at( + restored_app, + SchedulerConfig::new(SUBMITTER), + receipt.next_batch_nonce, + ); + std::fs::remove_dir_all(backup).unwrap(); + + let mut seen = Vec::new(); + loop { + // One row per page forces a page split between the batch at B and the + // direct arriving later in that same block. + let page = client + .historical_l1_inputs(era, start, Some(1)) + .await + .unwrap(); + assert_eq!(page.era_id, era); + assert_eq!(page.l1_stop_block, STOP); + assert_eq!(page.end_input_index, 8); + for item in page.items { + let source = &inputs[usize::try_from(item.input_index).unwrap()]; + assert_eq!(item.payload.as_ref(), source.payload); + assert_eq!(item.sender, source.sender); + assert_eq!(item.block_timestamp, source.block_timestamp); + assert_eq!(item.transaction_hash, source.transaction_hash); + seen.push(item.input_index); + if item.block_number <= receipt.inclusion_block { + if item.sender != metadata.deployment.batch_submitter_address { + replay.enqueue_direct(item.sender, item.block_number, item.payload.to_vec()); + } + } else { + let result = replay + .process_input(SchedulerInput { + sender: item.sender, + inclusion_block: item.block_number, + domain: domain(), + payload: item.payload.to_vec(), + }) + .unwrap(); + if item.input_index == 5 { + assert_eq!( + result.outcome, + ProcessOutcome::BatchRejected(BatchRejectReason::DecodeFailed) + ); + assert_eq!( + result.outputs.len(), + 3, + "malformed batch still drains overdue directs" + ); + assert_eq!(replay.next_expected_batch_nonce(), 1); + } + history.extend(notices(result.outputs)); + } + } + if page.next_input_index == page.end_input_index { + break; + } + start = HistoricalL1InputStart::NextInputIndex(page.next_input_index); + } + assert_eq!(seen, vec![1, 2, 3, 4, 5, 6, 7]); + assert_eq!( + history.len(), + 6, + "the young final direct waits until raw EOF" + ); + assert_eq!(replay.queued_direct_len(), 1); + history.extend(notices( + replay + .drain_covered_at(metadata.baseline.l1_stop_block) + .unwrap(), + )); + let (mut reader_app, nonce) = replay.finish(); + assert_eq!(nonce, metadata.baseline.next_batch_nonce); + assert_eq!( + reader_app.executed_input_count(), + metadata.history.available_from + ); + assert_eq!( + reader_app.canonical_snapshot_bytes().unwrap(), + reference_app.canonical_snapshot_bytes().unwrap() + ); + assert_eq!(history, expected_history); + let eof = client + .historical_l1_inputs(era, HistoricalL1InputStart::NextInputIndex(8), None) + .await + .unwrap(); + assert!(eof.items.is_empty()); + assert_eq!(eof.next_input_index, eof.end_input_index); + + let current = client.history(Some(era), None).await.unwrap(); + assert_eq!(current.baseline, metadata.baseline); + let mut stream = client + .subscribe(HistoryClaim { + version: current.history.version, + next_input: reader_app.executed_input_count(), + }) + .await + .unwrap(); + let live = deposit(1245, user, 60); + storage + .append_ingested_safe_inputs_with_timestamp( + 1245, + live.block_timestamp, + std::slice::from_ref(&live), + SUBMITTER, + &default_protocol_timing(), + FrontierMode::Populate, + ) + .unwrap(); + let mut head = storage.open_state().unwrap().unwrap(); + let execution = execute_direct_input( + &mut reference_app, + &DirectInput { + sender: live.sender, + block_number: live.block_number, + payload: live.payload.clone(), + }, + ) + .unwrap(); + expected_history.extend(notices(execution.outputs)); + storage + .close_frame_only_with_executions( + &mut head, + 1245, + SafeInputRange::new(8, 9), + &[DirectInputExecution { + safe_input_index: 8, + executed_input_offset: execution.offset, + }], + ) + .unwrap(); + let message = tokio::time::timeout(Duration::from_secs(3), stream.next()) + .await + .unwrap() + .unwrap() + .unwrap(); + let Message::Text(text) = message else { + panic!("expected application input") + }; + let message: WsTxMessage = serde_json::from_str(text.as_str()).unwrap(); + let WsTxMessage::DirectInput { + offset, + sender, + block_number, + payload, + input_index, + batch_nonce, + .. + } = message + else { + panic!("expected post-baseline deposit") + }; + assert_eq!(offset, 7, "terminal-drained D4 is not delivered again"); + assert_eq!(input_index, 8); + assert_eq!(batch_nonce, 2); + let execution = execute_direct_input( + &mut reader_app, + &DirectInput { + sender: sender.parse().unwrap(), + block_number, + payload: alloy_primitives::hex::decode(payload).unwrap(), + }, + ) + .unwrap(); + assert_eq!(execution.offset, ExecutedInputCount::new(7)); + history.extend(notices(execution.outputs)); + assert_eq!(history, expected_history); + assert_eq!( + reader_app.canonical_snapshot_bytes().unwrap(), + reference_app.canonical_snapshot_bytes().unwrap() + ); + let fixed = client + .historical_l1_inputs(era, HistoricalL1InputStart::AfterBlock(STOP), None) + .await + .unwrap(); + assert!( + fixed.items.is_empty(), + "new L1 inputs never extend this era's historical prefix" + ); + assert_eq!(fixed.end_input_index, 8); + drop(stream); + server.stop().await; +} diff --git a/sequencer/src/integration_tests/historical_bootstrap/recovery_compatibility.rs b/sequencer/src/integration_tests/historical_bootstrap/recovery_compatibility.rs new file mode 100644 index 00000000..165ed235 --- /dev/null +++ b/sequencer/src/integration_tests/historical_bootstrap/recovery_compatibility.rs @@ -0,0 +1,403 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +use std::path::{Path, PathBuf}; +use std::time::SystemTime; + +use sequencer_core::application::{ExecutionOutcome, validate_and_execute_user_op}; +use sequencer_core::history::{HistoryVersion, RecoveryGeneration}; +use sequencer_core::user_op::SignedUserOp; +use sequencer_rust_client::{ + HistoryPolicyError, HistoryReadError, SubscribeError, SubscribeStream, +}; +use tokio::sync::oneshot; + +use super::*; +use crate::ingress::inclusion_lane::IncludedUserOp; +use crate::storage::WriteHead; +use crate::storage::test_helpers::local_batch_payload; + +struct Backup { + claim: HistoryClaim, + path: PathBuf, +} + +impl Backup { + fn save( + directory: &Path, + name: &str, + app: &mut WalletApp, + history: &[Vec], + version: HistoryVersion, + ) -> Self { + let claim = HistoryClaim { + version, + next_input: app.executed_input_count(), + }; + let path = directory.join(name); + std::fs::create_dir(&path).unwrap(); + app.create_dump(&path.join("wallet")).unwrap(); + std::fs::write( + path.join("projection.json"), + serde_json::to_vec(&(claim, history)).unwrap(), + ) + .unwrap(); + Self { claim, path } + } + + fn restore(&self) -> (WalletApp, Vec>) { + let app = WalletApp::from_dump(&self.path.join("wallet")).unwrap(); + let (claim, history): (HistoryClaim, Vec>) = + serde_json::from_slice(&std::fs::read(self.path.join("projection.json")).unwrap()) + .unwrap(); + assert_eq!(claim, self.claim); + assert_eq!(app.executed_input_count(), claim.next_input); + (app, history) + } +} + +fn append_transfer( + storage: &mut Storage, + head: &mut WriteHead, + app: &mut WalletApp, + history: &mut Vec>, + signer: &PrivateKeySigner, + amount: u64, +) { + let op = UserOp { + nonce: app.current_user_nonce(signer.address()), + max_fee: head.frame_fee, + data: Method::Transfer(Transfer { + to: RECIPIENT, + amount: U256::from(amount), + }) + .as_ssz_bytes() + .into(), + }; + let outcome = + validate_and_execute_user_op(app, signer.address(), &op, head.frame_fee, head.safe_block) + .unwrap(); + let ExecutionOutcome::Included(execution) = outcome else { + panic!("fixture transfer rejected: {outcome:?}") + }; + history.extend(notices(execution.outputs)); + let (respond_to, _response) = oneshot::channel(); + let included = IncludedUserOp { + pending: PendingUserOp { + signed: SignedUserOp { + sender: signer.address(), + signature: signer + .sign_hash_sync(&op.eip712_signing_hash(&domain())) + .unwrap(), + user_op: op, + }, + respond_to, + received_at: SystemTime::now(), + }, + executed_input_offset: execution.offset, + }; + storage + .append_executed_user_ops_chunk(head, &[included]) + .unwrap(); +} + +fn close_and_accept( + storage: &mut Storage, + head: &mut WriteHead, + app: &mut WalletApp, + dumps: &Path, + inclusion_block: u64, +) { + let index = head.batch_index; + let nonce = storage.batch_nonce(index).unwrap(); + let prefix = dumps.join(format!("accepted-{index}")); + dump_info::create_dump_dir_with_info( + app, + &prefix, + &dump_info::DumpInfo { + format_version: dump_info::FORMAT_VERSION, + next_batch_nonce: nonce + 1, + }, + ) + .unwrap(); + storage + .close_frame_and_batch_with_snapshot( + head, + head.safe_block, + &prefix, + index, + app.executed_input_count(), + ) + .unwrap(); + let payload = local_batch_payload(storage, nonce); + storage + .append_safe_inputs( + inclusion_block, + &[crate::storage::StoredSafeInput { + sender: SUBMITTER, + block_number: inclusion_block, + payload, + }], + SUBMITTER, + &default_protocol_timing(), + ) + .unwrap(); +} + +fn recover_tip(storage: &mut Storage, head: &mut WriteHead, safe_block: u64) { + let protocol = default_protocol_timing(); + storage + .append_safe_inputs(safe_block, &[], SUBMITTER, &protocol) + .unwrap(); + let invalidated = storage + .recover_aging_tip_for_recovery(head.batch_index, &protocol, crate::clock::unix_now_ms()) + .unwrap(); + assert_eq!(invalidated, vec![head.batch_index]); + *head = storage.open_state().unwrap().unwrap(); +} + +async fn replay_one(stream: &mut SubscribeStream, app: &mut WalletApp, history: &mut Vec>) { + let message = tokio::time::timeout(Duration::from_secs(3), stream.next()) + .await + .unwrap() + .unwrap() + .unwrap(); + let Message::Text(text) = message else { + panic!("expected application input") + }; + let message: WsTxMessage = serde_json::from_str(text.as_str()).unwrap(); + let WsTxMessage::UserOp { + offset, + sender, + nonce, + fee, + data, + safe_block, + .. + } = message + else { + panic!("expected transfer") + }; + assert_eq!(offset, app.executed_input_count().get()); + let op = UserOp { + nonce, + max_fee: fee, + data: alloy_primitives::hex::decode(data).unwrap().into(), + }; + let outcome = + validate_and_execute_user_op(app, sender.parse().unwrap(), &op, fee, safe_block).unwrap(); + let ExecutionOutcome::Included(execution) = outcome else { + panic!("feed transfer rejected: {outcome:?}") + }; + history.extend(notices(execution.outputs)); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn checkpoint_compatibility_survives_missed_recoveries_and_retries_admission_races() { + let db = temp_db("reader-checkpoint-compatibility"); + let dumps = tempfile::tempdir().unwrap(); + let signer: PrivateKeySigner = format!("{:064x}", 1).parse().unwrap(); + let mut app = WalletApp::new(config()); + let mut history = Vec::new(); + let baseline = dumps.path().join("genesis"); + dump_info::create_dump_dir_with_info( + &mut app, + &baseline, + &dump_info::DumpInfo { + format_version: dump_info::FORMAT_VERSION, + next_batch_nonce: 0, + }, + ) + .unwrap(); + let mut storage = Storage::initialize_for_command(&db.path, LifecycleCommand::Setup).unwrap(); + pin_test_deployment_identity(&mut storage, SUBMITTER); + storage + .complete_baseline_setup(&baseline, ExecutedInputCount::ZERO, 0, 0, false) + .unwrap(); + let mut head = storage + .initialize_open_state(0, SafeInputRange::empty_at(0)) + .unwrap(); + let directs = [ + deposit(5, signer.address(), 1_000_000_000_000_000), + deposit(6, RECIPIENT, 100), + ]; + storage + .append_ingested_safe_inputs_with_timestamp( + 10, + crate::clock::unix_now_ms() / 1000, + &directs, + SUBMITTER, + &default_protocol_timing(), + FrontierMode::Populate, + ) + .unwrap(); + let mut executions = Vec::new(); + for (index, direct) in directs.into_iter().enumerate() { + let execution = execute_direct_input( + &mut app, + &DirectInput { + sender: direct.sender, + payload: direct.payload, + block_number: direct.block_number, + }, + ) + .unwrap(); + history.extend(notices(execution.outputs)); + executions.push(DirectInputExecution { + safe_input_index: index as u64, + executed_input_offset: execution.offset, + }); + } + storage + .close_frame_only_with_executions(&mut head, 10, SafeInputRange::new(0, 2), &executions) + .unwrap(); + append_transfer(&mut storage, &mut head, &mut app, &mut history, &signer, 10); + close_and_accept(&mut storage, &mut head, &mut app, dumps.path(), 20); + let generation_zero = storage.history_state().unwrap().version; + let good_zero = Backup::save(dumps.path(), "g0-at3", &mut app, &history, generation_zero); + append_transfer(&mut storage, &mut head, &mut app, &mut history, &signer, 11); + let bad_zero = Backup::save(dumps.path(), "g0-at4", &mut app, &history, generation_zero); + + let server = Server::start(&db.path).await; + let client = SequencerClient::new(format!("http://{}", server.addr)).unwrap(); + let era = generation_zero.era_id; + recover_tip(&mut storage, &mut head, 1500); + (app, history) = good_zero.restore(); + let generation_one = storage.history_state().unwrap().version; + assert_eq!(generation_one.recovery_generation.get(), 1); + append_transfer(&mut storage, &mut head, &mut app, &mut history, &signer, 20); + let middle_one = Backup::save(dumps.path(), "g1-at4", &mut app, &history, generation_one); + append_transfer(&mut storage, &mut head, &mut app, &mut history, &signer, 21); + close_and_accept(&mut storage, &mut head, &mut app, dumps.path(), 1501); + let good_one = Backup::save(dumps.path(), "g1-at5", &mut app, &history, generation_one); + append_transfer(&mut storage, &mut head, &mut app, &mut history, &signer, 22); + let bad_one = Backup::save(dumps.path(), "g1-at6", &mut app, &history, generation_one); + + recover_tip(&mut storage, &mut head, 3000); + (app, history) = good_one.restore(); + for amount in [30, 31, 32] { + append_transfer( + &mut storage, + &mut head, + &mut app, + &mut history, + &signer, + amount, + ); + } + let current = client.history(Some(era), None).await.unwrap(); + assert_eq!(current.history.head.get(), 8); + assert_eq!(current.history.version.recovery_generation.get(), 2); + assert_eq!(current.compatibility, None); + let same = client + .history(Some(era), Some(RecoveryGeneration::new(2))) + .await + .unwrap(); + assert_eq!(same.compatibility.unwrap().preserved_input_count.get(), 8); + let future = Some(RecoveryGeneration::new(u64::MAX)); + assert!(matches!( + client.history(Some(era), future).await, + Err(HistoryReadError::Http { status: 400, .. }) + )); + let mut other_era_bytes = *era.as_bytes(); + other_era_bytes[0] ^= 1; + let other_era = sequencer_core::history::EraId::from_bytes(other_era_bytes).unwrap(); + assert!(matches!( + client.history(Some(other_era), future).await, + Err(HistoryReadError::History(HistoryPolicyError::EraChanged { current: version })) + if version == current.history.version + )); + + // Every backup is evaluated under its own saved generation. Latest-cut-only + // matching would incorrectly resurrect the discarded g0 transfer at offset 3. + let mut selected = None; + for (candidate, eligible, cut) in [ + (&good_zero, true, 3), + (&bad_zero, false, 3), + (&middle_one, true, 5), + (&good_one, true, 5), + (&bad_one, false, 5), + ] { + let info = client + .history(Some(era), Some(candidate.claim.version.recovery_generation)) + .await + .unwrap(); + let compatibility = info.compatibility.unwrap(); + assert_eq!( + compatibility.from_generation, + candidate.claim.version.recovery_generation + ); + assert_eq!(compatibility.preserved_input_count.get(), cut); + let survives = candidate.claim.next_input >= info.history.available_from + && candidate.claim.next_input <= compatibility.preserved_input_count; + assert_eq!(survives, eligible); + if survives + && selected.is_none_or(|previous: &Backup| { + previous.claim.next_input < candidate.claim.next_input + }) + { + selected = Some(candidate); + } + } + let selected = selected.unwrap(); + assert_eq!(selected.claim, good_one.claim); + let (mut reader_app, mut reader_history) = selected.restore(); + let mut stream = client + .subscribe(HistoryClaim { + version: current.history.version, + next_input: selected.claim.next_input, + }) + .await + .unwrap(); + for _ in 5..8 { + replay_one(&mut stream, &mut reader_app, &mut reader_history).await; + } + assert_eq!( + reader_app.canonical_snapshot_bytes().unwrap(), + app.canonical_snapshot_bytes().unwrap() + ); + assert_eq!( + reader_history, history, + "restored projection loses invalidated transfers and follows their replacements" + ); + drop(stream); + + let lookup = client + .history(Some(era), Some(good_one.claim.version.recovery_generation)) + .await + .unwrap(); + assert_eq!(lookup.compatibility.unwrap().preserved_input_count.get(), 5); + recover_tip(&mut storage, &mut head, 4500); + let stale = client + .subscribe(HistoryClaim { + version: lookup.history.version, + next_input: good_one.claim.next_input, + }) + .await; + assert!( + matches!(stale, Err(SubscribeError::History(HistoryPolicyError::StaleGeneration { current })) if current.recovery_generation.get() == 3) + ); + let fresh = client + .history(Some(era), Some(good_one.claim.version.recovery_generation)) + .await + .unwrap(); + assert_eq!(fresh.compatibility.unwrap().preserved_input_count.get(), 5); + (reader_app, reader_history) = good_one.restore(); + (app, history) = good_one.restore(); + let mut stream = client + .subscribe(HistoryClaim { + version: fresh.history.version, + next_input: good_one.claim.next_input, + }) + .await + .unwrap(); + append_transfer(&mut storage, &mut head, &mut app, &mut history, &signer, 40); + replay_one(&mut stream, &mut reader_app, &mut reader_history).await; + assert_eq!( + reader_app.canonical_snapshot_bytes().unwrap(), + app.canonical_snapshot_bytes().unwrap() + ); + assert_eq!(reader_history, history); + drop(stream); + server.stop().await; +} diff --git a/sequencer/src/integration_tests/mod.rs b/sequencer/src/integration_tests/mod.rs index 8319d5fd..3aba7cee 100644 --- a/sequencer/src/integration_tests/mod.rs +++ b/sequencer/src/integration_tests/mod.rs @@ -5,5 +5,6 @@ mod batch_submitter; mod chain_id_validation; mod common; mod e2e_sequencer; +mod historical_bootstrap; mod snapshot_endpoints; mod ws_broadcaster; diff --git a/sequencer/src/storage/egress.rs b/sequencer/src/storage/egress.rs index bb0f44ba..2f64adbe 100644 --- a/sequencer/src/storage/egress.rs +++ b/sequencer/src/storage/egress.rs @@ -1,7 +1,7 @@ // (c) Cartesi and individual authors (see AUTHORS) // SPDX-License-Identifier: Apache-2.0 (see LICENSE) -//! Application replay entries shared by catch-up and consumer history reads. +//! Application replay entries and the historical L1 prefix for consumer bootstrap. use alloy_primitives::{Address, B256}; use rusqlite::{Result, Row}; @@ -12,6 +12,8 @@ use super::convert::{i64_to_u16, i64_to_u32, i64_to_u64}; mod canonical; pub(crate) use canonical::HistoryReadError; +mod historical; +pub(crate) use historical::HistoricalReadError; #[derive(Debug, Clone)] pub(crate) enum L2TxContext { diff --git a/sequencer/src/storage/egress/historical.rs b/sequencer/src/storage/egress/historical.rs new file mode 100644 index 00000000..730879a0 --- /dev/null +++ b/sequencer/src/storage/egress/historical.rs @@ -0,0 +1,262 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +//! Era-pinned raw L1 pages and their immutable handoff to application history. + +use alloy_primitives::{Address, B256}; +use rusqlite::{Connection, OptionalExtension, params}; +use sequencer_core::history::{ + EraId, ExecutedInputCount, HistoryBounds, HistoryPolicyError, RecoveryGeneration, +}; +use sequencer_core::history_api::{ + AcceptedCheckpoint, HISTORICAL_INPUT_MAX_ITEMS, HISTORICAL_INPUT_PAYLOAD_TARGET_BYTES, + HistoricalL1Input, HistoricalL1InputStart, HistoricalL1InputsPage, HistoryBaseline, + HistoryCompatibility, HistoryDeployment, HistoryInfo, +}; + +use crate::storage::Storage; +use crate::storage::convert::{i64_to_u64, u64_to_i64}; +use crate::storage::history::{ + next_executed_input_count_in, preserved_input_count_in, query_history_state, +}; +use crate::storage::l1_inputs::query_deployment_identity; +use crate::storage::mutations::batch_tree_anchor_in; +use crate::storage::safe_accepted_batches::canonical_divergence_in; +use crate::storage::snapshot_dumps::{finalized_dump_in, has_rollback_safe_snapshot_in}; + +#[derive(Debug, thiserror::Error)] +pub(crate) enum HistoricalReadError { + #[error(transparent)] + Policy(#[from] HistoryPolicyError), + #[error("{0}")] + BadRequest(String), + #[error("canonical divergence prevents accepted checkpoint selection")] + CanonicalDivergence, + #[error("reading historical L1 inputs: {0}")] + Storage(#[from] rusqlite::Error), +} + +impl Storage { + pub(crate) fn history_info( + &mut self, + expected_era: Option, + from_generation: Option, + ) -> Result { + self.read(|tx| { + let state = query_history_state(tx)?; + if expected_era.is_some_and(|era| era != state.version.era_id) { + return Ok(Err(HistoryPolicyError::EraChanged { + current: state.version, + } + .into())); + } + if from_generation.is_some() && expected_era.is_none() { + return Ok(Err(HistoricalReadError::BadRequest( + "era_id is required with from_generation".to_owned(), + ))); + } + if from_generation.is_some_and(|from| from > state.version.recovery_generation) { + return Ok(Err(HistoricalReadError::BadRequest( + "from_generation exceeds the current recovery generation".to_owned(), + ))); + } + if canonical_divergence_in(tx)?.is_some() { + return Ok(Err(HistoricalReadError::CanonicalDivergence)); + } + let deployment = + query_deployment_identity(tx)?.ok_or(rusqlite::Error::QueryReturnedNoRows)?; + let accepted = finalized_dump_in(tx)?; + if accepted.is_none() { + assert!( + has_rollback_safe_snapshot_in(tx)?, + "history has no rollback-safe snapshot" + ); + } + let head = next_executed_input_count_in(tx)?; + let compatibility = from_generation + .map(|from| { + preserved_input_count_in(tx, from, state.version.recovery_generation, head).map( + |preserved_input_count| HistoryCompatibility { + from_generation: from, + preserved_input_count, + }, + ) + }) + .transpose()?; + Ok(Ok(HistoryInfo { + deployment: HistoryDeployment { + chain_id: deployment.chain_id, + app_address: deployment.app_address, + input_box_address: deployment.input_box_address, + app_deployment_block: deployment.app_deployment_block, + batch_submitter_address: deployment.batch_submitter_address, + }, + history: HistoryBounds { + version: state.version, + available_from: ExecutedInputCount::new(state.base_executed_input_count), + head, + }, + baseline: HistoryBaseline { + l1_stop_block: state.base_safe_block, + l1_end_input_index: historical_end_in(tx, state.base_safe_block)?, + next_batch_nonce: batch_tree_anchor_in(tx)?, + }, + accepted_checkpoint: accepted.map(|snapshot| AcceptedCheckpoint { + inclusion_block: snapshot.inclusion_block, + executed_input_count: snapshot.executed_input_count, + next_batch_nonce: snapshot.next_batch_nonce, + }), + compatibility, + })) + })? + } + + pub(crate) fn historical_l1_inputs( + &mut self, + era: EraId, + start: HistoricalL1InputStart, + limit: usize, + ) -> Result { + self.read(|tx| { + let state = query_history_state(tx)?; + if era != state.version.era_id { + return Ok(Err(HistoryPolicyError::EraChanged { + current: state.version, + } + .into())); + } + let end = historical_end_in(tx, state.base_safe_block)?; + if !(1..=HISTORICAL_INPUT_MAX_ITEMS).contains(&limit) { + return Ok(Err(HistoricalReadError::BadRequest(format!( + "limit must be between 1 and {HISTORICAL_INPUT_MAX_ITEMS}" + )))); + } + let next = match start { + HistoricalL1InputStart::NextInputIndex(next) if next <= end => next, + HistoricalL1InputStart::AfterBlock(block) if block <= state.base_safe_block => { + first_input_after_block_in(tx, block, state.base_safe_block)?.unwrap_or(end) + } + HistoricalL1InputStart::NextInputIndex(_) => { + return Ok(Err(HistoricalReadError::BadRequest(format!( + "next_input_index exceeds historical end {end}" + )))); + } + HistoricalL1InputStart::AfterBlock(_) => { + return Ok(Err(HistoricalReadError::BadRequest(format!( + "after_block exceeds historical stop block {}", + state.base_safe_block + )))); + } + }; + raw_page_in(tx, era, state.base_safe_block, end, next, limit).map(Ok) + })? + } +} + +fn historical_end_in(conn: &Connection, stop: u64) -> rusqlite::Result { + let last: Option = conn + .query_row( + "SELECT safe_input_index FROM safe_inputs WHERE block_number <= ?1 \ + ORDER BY block_number DESC, safe_input_index DESC LIMIT 1", + [u64_to_i64(stop)], + |row| row.get(0), + ) + .optional()?; + Ok(last.map_or(0, |index| { + i64_to_u64(index) + .checked_add(1) + .expect("historical input index overflow") + })) +} + +fn first_input_after_block_in( + conn: &Connection, + block: u64, + stop: u64, +) -> rusqlite::Result> { + conn.query_row( + "SELECT safe_input_index FROM safe_inputs WHERE block_number > ?1 AND block_number <= ?2 \ + ORDER BY block_number, safe_input_index LIMIT 1", + params![u64_to_i64(block), u64_to_i64(stop)], + |row| Ok(i64_to_u64(row.get(0)?)), + ) + .optional() +} + +fn raw_page_in( + conn: &Connection, + era_id: EraId, + stop: u64, + end: u64, + mut next: u64, + limit: usize, +) -> rusqlite::Result { + let expected_len = (end - next).min(limit as u64); + let mut items = Vec::new(); + if expected_len > 0 { + let mut stmt = conn.prepare_cached( + "SELECT safe_input_index, sender, payload, block_number, block_timestamp, \ + transaction_hash, length(payload) \ + FROM safe_inputs WHERE safe_input_index >= ?1 AND safe_input_index <= ?2 \ + ORDER BY safe_input_index LIMIT ?3", + )?; + let mut rows = stmt.query(params![ + u64_to_i64(next), + u64_to_i64(end - 1), + u64_to_i64(expected_len), + ])?; + let mut payload_bytes = 0_u64; + let mut byte_limited = false; + while let Some(row) = rows.next()? { + let input_index = i64_to_u64(row.get(0)?); + assert_eq!(input_index, next, "historical L1 page has an input gap"); + let payload_len = i64_to_u64(row.get(6)?); + // Budget before copying the BLOB out of SQLite. The first row may + // exceed the target so every valid L1 input remains consumable. + if !items.is_empty() + && payload_len > (HISTORICAL_INPUT_PAYLOAD_TARGET_BYTES as u64 - payload_bytes) + { + byte_limited = true; + break; + } + let block_number = i64_to_u64(row.get(3)?); + assert!( + block_number <= stop, + "historical L1 page exceeds its stop block" + ); + items.push(HistoricalL1Input { + input_index, + sender: Address::from_slice(&row.get::<_, Vec>(1)?), + payload: row.get::<_, Vec>(2)?.into(), + block_number, + block_timestamp: i64_to_u64(row.get(4)?), + transaction_hash: B256::from_slice(&row.get::<_, Vec>(5)?), + }); + next = next + .checked_add(1) + .expect("historical input index overflow"); + payload_bytes += payload_len; + if payload_bytes > HISTORICAL_INPUT_PAYLOAD_TARGET_BYTES as u64 { + byte_limited = true; + break; + } + } + if !byte_limited { + assert_eq!( + items.len() as u64, + expected_len, + "historical L1 page ended before its recorded end" + ); + } + } + Ok(HistoricalL1InputsPage { + era_id, + l1_stop_block: stop, + end_input_index: end, + next_input_index: next, + items, + }) +} + +#[cfg(test)] +mod tests; diff --git a/sequencer/src/storage/egress/historical/tests.rs b/sequencer/src/storage/egress/historical/tests.rs new file mode 100644 index 00000000..cf1587d7 --- /dev/null +++ b/sequencer/src/storage/egress/historical/tests.rs @@ -0,0 +1,495 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +use std::path::Path; + +use super::*; +use crate::storage::history::{advance_recovery_generation_in, initialize_history_in}; +use crate::storage::test_helpers::{ + SENDER_A, SENDER_B, TestDb, default_protocol_timing, local_batch_payload, + pin_test_deployment_identity, temp_db, +}; +use crate::storage::{ + DirectInputExecution, FrontierMode, IngestedSafeInput, LifecycleCommand, SafeInputRange, +}; + +fn fixture(name: &str, stop: u64, count: u64, nonce: u64) -> (TestDb, Storage) { + let db = temp_db(name); + let mut storage = Storage::initialize_for_command(&db.path, LifecycleCommand::Rebuild).unwrap(); + pin_test_deployment_identity(&mut storage, SENDER_A); + storage + .write(|tx| initialize_history_in(tx, ExecutedInputCount::new(count), stop)) + .unwrap(); + storage.set_batch_tree_anchor(nonce).unwrap(); + storage + .insert_baseline_snapshot( + Path::new("/snapshot/baseline"), + ExecutedInputCount::new(count), + ) + .unwrap(); + (db, storage) +} + +fn input(block: u64, sender: Address, payload: Vec) -> IngestedSafeInput { + IngestedSafeInput { + sender, + payload, + block_number: block, + block_timestamp: block * 12, + transaction_hash: B256::repeat_byte(block as u8), + } +} + +fn append(storage: &mut Storage, safe_block: u64, inputs: &[IngestedSafeInput]) { + storage + .append_ingested_safe_inputs_with_timestamp( + safe_block, + safe_block * 12, + inputs, + SENDER_A, + &default_protocol_timing(), + FrontierMode::DeferUntilAnchorSet, + ) + .unwrap(); +} + +fn era(storage: &Storage) -> EraId { + storage.history_state().unwrap().version.era_id +} + +fn page(storage: &mut Storage, next: u64, limit: usize) -> HistoricalL1InputsPage { + storage + .historical_l1_inputs( + era(storage), + HistoricalL1InputStart::NextInputIndex(next), + limit, + ) + .unwrap() +} + +#[test] +fn genesis_history_has_an_empty_raw_prefix_even_after_live_inputs_arrive() { + let (_db, mut storage) = fixture("historical-genesis", 0, 0, 0); + append(&mut storage, 20, &[input(10, SENDER_B, vec![1])]); + let info = storage.history_info(None, None).unwrap(); + assert_eq!(info.history.available_from, ExecutedInputCount::ZERO); + assert_eq!(info.history.head, ExecutedInputCount::ZERO); + assert_eq!(info.compatibility, None); + assert_eq!( + info.baseline, + HistoryBaseline { + l1_stop_block: 0, + l1_end_input_index: 0, + next_batch_nonce: 0 + } + ); + assert_eq!( + info.accepted_checkpoint, + Some(AcceptedCheckpoint { + inclusion_block: 0, + executed_input_count: ExecutedInputCount::ZERO, + next_batch_nonce: 0, + }) + ); + assert_eq!(info.deployment.chain_id, 1); + assert_eq!(info.deployment.app_address, Address::repeat_byte(0x11)); + assert_eq!( + info.deployment.input_box_address, + Address::repeat_byte(0x22) + ); + assert_eq!(info.deployment.app_deployment_block, 0); + assert_eq!(info.deployment.batch_submitter_address, SENDER_A); + let raw = page(&mut storage, 0, 10); + assert!(raw.items.is_empty()); + assert_eq!((raw.end_input_index, raw.next_input_index), (0, 0)); +} + +#[test] +fn rebuilt_history_preserves_raw_payloads_metadata_and_its_fixed_prefix() { + let (_db, mut storage) = fixture("historical-rebuilt", 20, 41, 7); + let inputs = [ + input(5, SENDER_B, vec![0xaa, 0xbb]), + input(12, SENDER_A, vec![0xff]), // Deliberately malformed batch bytes. + input(20, SENDER_B, vec![]), + input(21, SENDER_B, vec![0xcc]), + ]; + append(&mut storage, 25, &inputs); + let info = storage.history_info(None, None).unwrap(); + assert_eq!(info.history.available_from, ExecutedInputCount::new(41)); + assert_eq!(info.history.head, ExecutedInputCount::new(41)); + assert_eq!(info.baseline.l1_stop_block, 20); + assert_eq!(info.baseline.l1_end_input_index, 3); + assert_eq!(info.baseline.next_batch_nonce, 7); + assert_eq!(info.accepted_checkpoint, None); + + let first = page(&mut storage, 0, 2); + assert_eq!((first.end_input_index, first.next_input_index), (3, 2)); + for (index, actual) in first.items.iter().enumerate() { + let expected = &inputs[index]; + assert_eq!(actual.input_index, index as u64); + assert_eq!(actual.sender, expected.sender); + assert_eq!(actual.payload.as_ref(), expected.payload); + assert_eq!(actual.block_number, expected.block_number); + assert_eq!(actual.block_timestamp, expected.block_timestamp); + assert_eq!(actual.transaction_hash, expected.transaction_hash); + } + let last = page(&mut storage, first.next_input_index, 2); + assert_eq!(last.items.len(), 1); + assert_eq!(last.items[0].input_index, 2); + assert!(last.items[0].payload.is_empty()); + assert_eq!((last.end_input_index, last.next_input_index), (3, 3)); + assert!(page(&mut storage, 3, 2).items.is_empty()); + + append(&mut storage, 40, &[input(35, SENDER_B, vec![9])]); + storage.write(advance_recovery_generation_in).unwrap(); + assert_eq!(page(&mut storage, 0, 2), first); + let updated = storage + .history_info(Some(info.history.version.era_id), None) + .unwrap(); + assert_eq!(updated.baseline, info.baseline); + assert_eq!(updated.history.version.recovery_generation.get(), 1); +} + +#[test] +fn after_block_skips_the_whole_block_and_paging_keeps_its_remaining_rows() { + let (_db, mut storage) = fixture("historical-block-seek", 30, 9, 2); + append( + &mut storage, + 40, + &[ + input(5, SENDER_B, vec![0]), + input(12, SENDER_A, vec![1]), + input(12, SENDER_B, vec![2]), + input(20, SENDER_B, vec![3]), + input(20, SENDER_A, vec![4]), + input(31, SENDER_B, vec![5]), + ], + ); + let current = era(&storage); + let first = storage + .historical_l1_inputs(current, HistoricalL1InputStart::AfterBlock(12), 1) + .unwrap(); + assert_eq!(first.items[0].input_index, 3); + assert_eq!(first.next_input_index, 4); + assert_eq!(page(&mut storage, 4, 1).items[0].input_index, 4); + for block in [20, 25, 30] { + let eof = storage + .historical_l1_inputs(current, HistoricalL1InputStart::AfterBlock(block), 1) + .unwrap(); + assert!(eof.items.is_empty()); + assert_eq!((eof.end_input_index, eof.next_input_index), (5, 5)); + } +} + +#[test] +fn era_validation_precedes_numeric_bounds_even_for_empty_history() { + let (_db, mut storage) = fixture("historical-claim", 0, 0, 0); + let current = storage.history_state().unwrap().version; + let other: EraId = "00112233-4455-4677-8899-aabbccddeeff".parse().unwrap(); + assert_ne!(other, current.era_id); + for start in [ + HistoricalL1InputStart::NextInputIndex(0), + HistoricalL1InputStart::NextInputIndex(u64::MAX), + HistoricalL1InputStart::AfterBlock(u64::MAX), + ] { + for limit in [0, 1, usize::MAX] { + assert!(matches!( + storage.historical_l1_inputs(other, start, limit), + Err(HistoricalReadError::Policy(HistoryPolicyError::EraChanged { current: actual })) + if actual == current + )); + } + } + assert!(matches!( + storage.history_info(Some(other), None), + Err(HistoricalReadError::Policy(HistoryPolicyError::EraChanged { current: actual })) + if actual == current + )); + for (start, limit) in [ + (HistoricalL1InputStart::NextInputIndex(0), 0), + ( + HistoricalL1InputStart::NextInputIndex(0), + HISTORICAL_INPUT_MAX_ITEMS + 1, + ), + (HistoricalL1InputStart::NextInputIndex(1), 1), + (HistoricalL1InputStart::NextInputIndex(u64::MAX), 1), + (HistoricalL1InputStart::AfterBlock(1), 1), + (HistoricalL1InputStart::AfterBlock(u64::MAX), 1), + ] { + assert!(matches!( + storage.historical_l1_inputs(current.era_id, start, limit), + Err(HistoricalReadError::BadRequest(_)) + )); + } +} + +#[test] +fn byte_budget_makes_progress_through_oversized_inputs_without_skipping_them() { + let (_db, mut storage) = fixture("historical-byte-budget", 20, 1, 1); + let target = HISTORICAL_INPUT_PAYLOAD_TARGET_BYTES; + append( + &mut storage, + 20, + &[ + input(1, SENDER_B, vec![1; target / 2]), + input(2, SENDER_B, vec![2; target / 2 + 1]), + input(3, SENDER_A, vec![3; target + 1]), + input(4, SENDER_B, vec![4]), + ], + ); + for (start, expected_len) in [ + (0, target / 2), + (1, target / 2 + 1), + (2, target + 1), + (3, 1), + ] { + let raw = page(&mut storage, start, HISTORICAL_INPUT_MAX_ITEMS); + assert_eq!(raw.items.len(), 1); + assert_eq!(raw.items[0].input_index, start); + assert_eq!(raw.items[0].payload.len(), expected_len); + assert_eq!(raw.next_input_index, start + 1); + assert_eq!(raw.end_input_index, 4); + } +} + +#[test] +fn exact_byte_target_and_item_cap_have_independent_boundaries() { + let (_db, mut storage) = fixture("historical-item-budget", 20, 1, 1); + let mut inputs = vec![input( + 1, + SENDER_B, + vec![7; HISTORICAL_INPUT_PAYLOAD_TARGET_BYTES], + )]; + inputs.extend((0..HISTORICAL_INPUT_MAX_ITEMS).map(|_| input(2, SENDER_B, vec![]))); + append(&mut storage, 20, &inputs); + let first = page(&mut storage, 0, HISTORICAL_INPUT_MAX_ITEMS); + assert_eq!(first.items.len(), HISTORICAL_INPUT_MAX_ITEMS); + assert_eq!(first.next_input_index, HISTORICAL_INPUT_MAX_ITEMS as u64); + assert_eq!(first.end_input_index, HISTORICAL_INPUT_MAX_ITEMS as u64 + 1); + let last = page( + &mut storage, + first.next_input_index, + HISTORICAL_INPUT_MAX_ITEMS, + ); + assert_eq!(last.items.len(), 1); + assert_eq!(last.next_input_index, last.end_input_index); +} + +#[test] +#[should_panic(expected = "historical L1 page has an input gap")] +fn missing_raw_input_is_an_invariant_fault() { + let (_db, mut storage) = fixture("historical-gap", 20, 3, 1); + append( + &mut storage, + 20, + &[ + input(1, SENDER_B, vec![1]), + input(2, SENDER_B, vec![2]), + input(3, SENDER_B, vec![3]), + ], + ); + storage + .conn + .execute("DELETE FROM safe_inputs WHERE safe_input_index=1", []) + .unwrap(); + let _ = page(&mut storage, 0, 3); +} + +#[test] +fn maximum_sqlite_index_has_an_unclamped_exclusive_end() { + let (_db, mut storage) = fixture("historical-max-index", 20, 1, 1); + storage.conn.execute( + "INSERT INTO safe_inputs (safe_input_index,sender,payload,block_number,block_timestamp,transaction_hash) \ + VALUES (?1,?2,?3,20,240,?4)", + params![i64::MAX, SENDER_B.as_slice(), &[1_u8][..], B256::ZERO.as_slice()], + ).unwrap(); + let max = i64::MAX as u64; + let last = page(&mut storage, max, 10); + assert_eq!(last.items.len(), 1); + assert_eq!( + (last.end_input_index, last.next_input_index), + (max + 1, max + 1) + ); + assert!(page(&mut storage, max + 1, 10).items.is_empty()); +} + +#[test] +fn accepted_checkpoint_uses_the_exact_latest_snapshot_and_preserves_baseline() { + let (_db, mut storage) = fixture("historical-accepted", 20, 41, 7); + let mut head = storage + .initialize_open_state(20, SafeInputRange::empty_at(0)) + .unwrap(); + for index in 0..2 { + storage + .close_frame_and_batch_with_snapshot( + &mut head, + 20, + Path::new(&format!("/snapshot/{index}")), + index, + ExecutedInputCount::new(41), + ) + .unwrap(); + } + let payloads = [ + local_batch_payload(&mut storage, 7), + local_batch_payload(&mut storage, 8), + ]; + storage + .append_ingested_safe_inputs_with_timestamp( + 30, + 360, + &[ + input(29, SENDER_A, payloads[0].clone()), + input(30, SENDER_A, payloads[1].clone()), + ], + SENDER_A, + &default_protocol_timing(), + FrontierMode::Populate, + ) + .unwrap(); + storage.gc_unreferenced_dumps().unwrap(); + let info = storage.history_info(None, None).unwrap(); + assert_eq!(info.baseline.next_batch_nonce, 7); + assert_eq!(info.baseline.l1_stop_block, 20); + assert_eq!(info.baseline.l1_end_input_index, 0); + assert_eq!( + info.accepted_checkpoint, + Some(AcceptedCheckpoint { + inclusion_block: 30, + executed_input_count: ExecutedInputCount::new(41), + next_batch_nonce: 9, + }) + ); + storage + .conn + .execute("DELETE FROM snapshots WHERE batch_index=1", []) + .unwrap(); + assert!(matches!( + storage.history_info(None, None), + Err(HistoricalReadError::Storage( + rusqlite::Error::QueryReturnedNoRows + )) + )); +} + +#[test] +fn canonical_divergence_cannot_be_advertised_as_an_accepted_receipt() { + let (_db, mut storage) = fixture("historical-divergence", 0, 0, 0); + storage + .conn + .execute( + "INSERT INTO canonical_divergence \ + (singleton_id,nonce,safe_input_index,kind,detected_at_ms) VALUES (0,0,0,'foreign',0)", + [], + ) + .unwrap(); + assert!(matches!( + storage.history_info(None, None), + Err(HistoricalReadError::CanonicalDivergence) + )); +} + +#[test] +#[should_panic(expected = "history has no rollback-safe snapshot")] +fn missing_baseline_is_not_reported_as_an_absent_accepted_checkpoint() { + let (_db, mut storage) = fixture("historical-missing-baseline", 20, 41, 7); + storage.conn.execute("DELETE FROM snapshots", []).unwrap(); + let _ = storage.history_info(None, None); +} + +#[test] +fn compatibility_requires_an_era_and_rejects_future_generations_after_era_validation() { + let (_db, mut storage) = fixture("historical-generation-query", 0, 0, 0); + let current = era(&storage); + assert!(matches!( + storage.history_info(None, Some(RecoveryGeneration::new(0))), + Err(HistoricalReadError::BadRequest(_)) + )); + for from in [1, u64::MAX] { + assert!(matches!( + storage.history_info(Some(current), Some(RecoveryGeneration::new(from))), + Err(HistoricalReadError::BadRequest(_)) + )); + } + let other = "00112233-4455-4677-8899-aabbccddeeff".parse().unwrap(); + assert!(matches!( + storage.history_info(Some(other), Some(RecoveryGeneration::new(u64::MAX))), + Err(HistoricalReadError::Policy( + HistoryPolicyError::EraChanged { .. } + )) + )); + let info = storage + .history_info(Some(current), Some(RecoveryGeneration::new(0))) + .unwrap(); + assert_eq!( + info.compatibility, + Some(HistoryCompatibility { + from_generation: RecoveryGeneration::new(0), + preserved_input_count: ExecutedInputCount::ZERO, + }) + ); +} + +#[test] +fn full_recovery_preserves_nonzero_baseline_before_replacement_directs() { + let (_db, mut storage) = fixture("historical-generation-baseline", 100, 41, 7); + let mut head = storage + .initialize_open_state(100, SafeInputRange::empty_at(0)) + .unwrap(); + let now = crate::clock::unix_now_ms(); + let protocol = default_protocol_timing(); + storage + .append_ingested_safe_inputs_with_timestamp( + 1400, + now / 1000, + &[input(110, SENDER_B, vec![1])], + SENDER_A, + &protocol, + FrontierMode::Populate, + ) + .unwrap(); + storage + .close_frame_only_with_executions( + &mut head, + 110, + SafeInputRange::new(0, 1), + &[DirectInputExecution { + safe_input_index: 0, + executed_input_offset: ExecutedInputCount::new(41), + }], + ) + .unwrap(); + let current = era(&storage); + let before = storage + .history_info(Some(current), Some(RecoveryGeneration::new(0))) + .unwrap(); + assert_eq!(before.history.head, ExecutedInputCount::new(42)); + assert_eq!( + before.compatibility.unwrap().preserved_input_count, + before.history.head + ); + assert_eq!( + storage + .recover_aging_tip_for_recovery(head.batch_index, &protocol, now) + .unwrap(), + [0] + ); + + let after = storage + .history_info(Some(current), Some(RecoveryGeneration::new(0))) + .unwrap(); + assert_eq!(after.history.available_from, ExecutedInputCount::new(41)); + assert_eq!(after.history.head, ExecutedInputCount::new(42)); + assert_eq!(after.history.version.recovery_generation.get(), 1); + assert_eq!( + after.compatibility.unwrap().preserved_input_count, + ExecutedInputCount::new(41) + ); + let latest = storage + .history_info(Some(current), Some(RecoveryGeneration::new(1))) + .unwrap(); + assert_eq!( + latest.compatibility.unwrap().preserved_input_count, + latest.history.head + ); +} diff --git a/sequencer/src/storage/history.rs b/sequencer/src/storage/history.rs index 65f24f37..0948d1db 100644 --- a/sequencer/src/storage/history.rs +++ b/sequencer/src/storage/history.rs @@ -1,7 +1,7 @@ // (c) Cartesi and individual authors (see AUTHORS) // SPDX-License-Identifier: Apache-2.0 (see LICENSE) -//! Immutable era baseline and current application-history generation. +//! Immutable era baseline and the preserved prefix at each recovery generation. #[cfg(test)] use rusqlite::OptionalExtension; @@ -112,11 +112,18 @@ pub(super) fn next_executed_input_count_in(conn: &Connection) -> Result) -> Result { let current = query_history_state(tx)?.version.recovery_generation.get(); let next = current .checked_add(1) .expect("recovery generation exhausted"); + let preserved = next_executed_input_count_in(tx)?; + tx.execute( + "INSERT INTO history_generation_cuts (recovery_generation, preserved_input_count) \ + VALUES (?1, ?2)", + params![u64_to_i64(next), u64_to_i64(preserved.get())], + )?; let changed = tx.execute( "UPDATE history_state SET recovery_generation = ?1 WHERE singleton_id = 0", [u64_to_i64(next)], @@ -127,6 +134,36 @@ pub(super) fn advance_recovery_generation_in(tx: &Transaction<'_>) -> Result Result { + assert!( + from <= current, + "compatibility starts after the current generation" + ); + if from == current { + return Ok(head); + } + let (count, minimum): (i64, Option) = conn.query_row( + "SELECT COUNT(*), MIN(preserved_input_count) FROM history_generation_cuts \ + WHERE recovery_generation > ?1 AND recovery_generation <= ?2", + params![u64_to_i64(from.get()), u64_to_i64(current.get())], + |row| Ok((row.get(0)?, row.get(1)?)), + )?; + // Unique integer generations plus the exact interval length prove that + // every intervening recovery contributed its cut, including empty batches. + assert_eq!( + i64_to_u64(count), + current.get() - from.get(), + "history generation lineage is incomplete" + ); + let minimum = minimum.expect("a nonempty complete generation interval has a minimum"); + Ok(head.min(ExecutedInputCount::new(i64_to_u64(minimum)))) +} + #[cfg(test)] mod tests { use super::*; @@ -194,3 +231,6 @@ mod tests { ); } } + +#[cfg(test)] +mod generation_tests; diff --git a/sequencer/src/storage/history/generation_tests.rs b/sequencer/src/storage/history/generation_tests.rs new file mode 100644 index 00000000..977db217 --- /dev/null +++ b/sequencer/src/storage/history/generation_tests.rs @@ -0,0 +1,170 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +use super::*; +use crate::storage::test_helpers::temp_db; + +fn ledger(storage: &Storage) -> Vec<(i64, i64)> { + storage + .conn + .prepare("SELECT recovery_generation, preserved_input_count FROM history_generation_cuts ORDER BY recovery_generation") + .unwrap() + .query_map([], |row| Ok((row.get(0)?, row.get(1)?))) + .unwrap() + .collect::>() + .unwrap() +} + +// Synthetic cuts exercise the query's full interval contract independently +// of today's recovery pivot policy. Actual cascades are tested separately. +fn seed_cuts(storage: &mut Storage, cuts: &[u64]) { + storage + .write(|tx| { + for (index, cut) in cuts.iter().enumerate() { + let generation = i64::try_from(index + 1).unwrap(); + tx.execute( + "INSERT INTO history_generation_cuts (recovery_generation,preserved_input_count) VALUES (?1,?2)", + params![generation, u64_to_i64(*cut)], + )?; + tx.execute( + "UPDATE history_state SET recovery_generation=?1 WHERE singleton_id=0", + [generation], + )?; + } + Ok(()) + }) + .unwrap(); +} + +#[test] +fn generation_and_cut_are_atomic_and_immutable() { + let db = temp_db("generation-cut-atomic"); + let mut storage = Storage::open(&db.path).unwrap(); + let result: Result<()> = storage.write(|tx| { + advance_recovery_generation_in(tx)?; + Err(rusqlite::Error::InvalidQuery) + }); + assert!(result.is_err()); + assert!(ledger(&storage).is_empty()); + assert_eq!( + storage + .history_state() + .unwrap() + .version + .recovery_generation + .get(), + 0 + ); + assert!( + storage + .conn + .execute("UPDATE history_state SET recovery_generation=1", []) + .is_err() + ); + + storage.write(advance_recovery_generation_in).unwrap(); + assert_eq!(ledger(&storage), [(1, 0)]); + for sql in [ + "UPDATE history_generation_cuts SET preserved_input_count=1", + "UPDATE history_generation_cuts SET recovery_generation=2", + "DELETE FROM history_generation_cuts", + "INSERT INTO history_generation_cuts VALUES (1,1)", + "INSERT INTO history_generation_cuts VALUES (0,0)", + "INSERT INTO history_generation_cuts VALUES (2,-1)", + ] { + assert!(storage.conn.execute(sql, []).is_err(), "{sql}"); + } + drop(storage); + let reopened = Storage::open(&db.path).unwrap(); + assert_eq!(ledger(&reopened), [(1, 0)]); + assert_eq!( + reopened + .history_state() + .unwrap() + .version + .recovery_generation + .get(), + 1 + ); +} + +#[test] +fn compatibility_uses_every_intervening_cut_and_the_current_head() { + let db = temp_db("generation-cut-minimum"); + let mut storage = Storage::open(&db.path).unwrap(); + seed_cuts(&mut storage, &[3, 5, 2, 4]); + for (from, current, head, expected) in [ + (0, 1, 9, 3), + (0, 2, 9, 3), + (1, 2, 9, 5), + (0, 3, 9, 2), + (1, 3, 9, 2), + (2, 3, 9, 2), + (0, 4, 9, 2), + (3, 4, 9, 4), + (3, 4, 1, 1), + (4, 4, 9, 9), + ] { + assert_eq!( + preserved_input_count_in( + &storage.conn, + RecoveryGeneration::new(from), + RecoveryGeneration::new(current), + ExecutedInputCount::new(head) + ) + .unwrap(), + ExecutedInputCount::new(expected), + "from={from}, current={current}, head={head}", + ); + } +} + +#[test] +#[should_panic(expected = "history generation lineage is incomplete")] +fn missing_intermediate_cut_fails_instead_of_certifying_a_partial_minimum() { + let db = temp_db("generation-cut-gap"); + let mut storage = Storage::open(&db.path).unwrap(); + seed_cuts(&mut storage, &[3, 1, 5]); + storage + .conn + .execute_batch( + "DROP TRIGGER trg_history_generation_cuts_not_deletable; \ + DELETE FROM history_generation_cuts WHERE recovery_generation=2", + ) + .unwrap(); + let _ = preserved_input_count_in( + &storage.conn, + RecoveryGeneration::new(0), + RecoveryGeneration::new(3), + ExecutedInputCount::new(9), + ); +} + +#[test] +fn current_head_may_be_the_boundary_after_the_maximum_sqlite_offset() { + let db = temp_db("generation-cut-max-head"); + let mut storage = Storage::open(&db.path).unwrap(); + let max = i64::MAX as u64; + seed_cuts(&mut storage, &[max]); + let head = ExecutedInputCount::new(max + 1); + assert_eq!( + preserved_input_count_in( + &storage.conn, + RecoveryGeneration::new(1), + RecoveryGeneration::new(1), + head + ) + .unwrap(), + head, + ); + assert_eq!( + preserved_input_count_in( + &storage.conn, + RecoveryGeneration::new(0), + RecoveryGeneration::new(1), + head + ) + .unwrap(), + ExecutedInputCount::new(max), + ); +} diff --git a/sequencer/src/storage/migrations/0001_schema.sql b/sequencer/src/storage/migrations/0001_schema.sql index a9650de0..f93e6914 100644 --- a/sequencer/src/storage/migrations/0001_schema.sql +++ b/sequencer/src/storage/migrations/0001_schema.sql @@ -376,6 +376,19 @@ CREATE TABLE IF NOT EXISTS history_state ( base_safe_block INTEGER NOT NULL CHECK ( typeof(base_safe_block) = 'integer' AND base_safe_block >= 0) ); +-- One cut for each transition, measured after invalidation removes the old +-- suffix and before reopening attributes replacement direct inputs. +CREATE TABLE IF NOT EXISTS history_generation_cuts ( + recovery_generation INTEGER PRIMARY KEY CHECK (recovery_generation > 0), + preserved_input_count INTEGER NOT NULL CHECK ( + typeof(preserved_input_count) = 'integer' AND preserved_input_count >= 0) +); +CREATE TRIGGER IF NOT EXISTS trg_history_generation_cuts_immutable +BEFORE UPDATE ON history_generation_cuts FOR EACH ROW +BEGIN SELECT RAISE(ABORT, 'history generation cuts are immutable'); END; +CREATE TRIGGER IF NOT EXISTS trg_history_generation_cuts_not_deletable +BEFORE DELETE ON history_generation_cuts FOR EACH ROW +BEGIN SELECT RAISE(ABORT, 'history generation cuts are retained throughout the era'); END; CREATE TRIGGER IF NOT EXISTS trg_history_state_single_insert BEFORE INSERT ON history_state FOR EACH ROW WHEN EXISTS (SELECT 1 FROM history_state) @@ -388,7 +401,9 @@ CREATE TRIGGER IF NOT EXISTS trg_history_generation_monotonic BEFORE UPDATE OF recovery_generation ON history_state FOR EACH ROW WHEN OLD.recovery_generation = 9223372036854775807 OR NEW.recovery_generation != OLD.recovery_generation + 1 -BEGIN SELECT RAISE(ABORT, 'recovery generation must advance by exactly one'); END; + OR NOT EXISTS (SELECT 1 FROM history_generation_cuts + WHERE recovery_generation = NEW.recovery_generation) +BEGIN SELECT RAISE(ABORT, 'recovery generation must advance by exactly one with a recorded cut'); END; CREATE TRIGGER IF NOT EXISTS trg_history_state_not_deletable BEFORE DELETE ON history_state FOR EACH ROW BEGIN SELECT RAISE(ABORT, 'history state is write-once per database'); END; diff --git a/sequencer/src/storage/mod.rs b/sequencer/src/storage/mod.rs index 9b1308dc..d7fa5332 100644 --- a/sequencer/src/storage/mod.rs +++ b/sequencer/src/storage/mod.rs @@ -54,7 +54,7 @@ use thiserror::Error; #[cfg(test)] pub(crate) use egress::ApplicationInputRow; -pub(crate) use egress::{HistoryReadError, L2TxContext}; +pub(crate) use egress::{HistoricalReadError, HistoryReadError, L2TxContext}; pub use history::{DirectInputExecution, HistoryState}; pub use lifecycle::{LifecycleCommand, LifecycleError, TerminalFault}; pub use open::Storage; diff --git a/sequencer/src/storage/recovery.rs b/sequencer/src/storage/recovery.rs index f09e5ebc..7d8d05de 100644 --- a/sequencer/src/storage/recovery.rs +++ b/sequencer/src/storage/recovery.rs @@ -478,9 +478,9 @@ fn recover_aging_tip_inner(tx: &Transaction<'_>, danger_threshold: u64) -> Resul /// state. Accepted batch snapshots survive; before any acceptance the /// baseline supplies the rollback-safe restore point. /// 3. **Advance `RecoveryGeneration`** exactly once when the cascade -/// invalidated any valid batch. This is the externally visible statement -/// that the current era's soft-history reality changed; composing it here -/// makes generation and invalidation inseparable across crashes. +/// invalidated any valid batch, recording the surviving application count +/// before replacement directs can reuse its offsets. Cut, generation, and +/// invalidation remain inseparable across crashes. /// 4. **Reopen the Tip** the cascade just invalidated (or one a torn crash /// left missing), atomically with the cascade. Same mechanism the /// runtime's genesis path uses — see `ingress::open_fresh_tip_in_tx`. diff --git a/sequencer/src/storage/recovery_tests.rs b/sequencer/src/storage/recovery_tests.rs index a690e026..30fc2687 100644 --- a/sequencer/src/storage/recovery_tests.rs +++ b/sequencer/src/storage/recovery_tests.rs @@ -11,6 +11,17 @@ use alloy_primitives::Address; use sequencer_core::l2_tx::SequencedL2Tx; use sequencer_core::protocol::ProtocolTiming; +fn generation_cuts(storage: &Storage) -> Vec<(i64, i64)> { + storage + .conn + .prepare("SELECT recovery_generation, preserved_input_count FROM history_generation_cuts ORDER BY recovery_generation") + .unwrap() + .query_map([], |row| Ok((row.get(0)?, row.get(1)?))) + .unwrap() + .collect::>() + .unwrap() +} + /// Exercise the same frame-advance-before-close sequence as the live lane. trait RecoveryFixture { fn close_batch_at( @@ -576,6 +587,11 @@ mod recover_post_flush { .expect("append safe input"); let first = storage.recover_post_flush(1200).expect("first detect"); assert_eq!(first, vec![0, 1]); + assert_eq!( + generation_cuts(&storage), + [(1, 0)], + "empty invalidated batches still record the old head" + ); assert_eq!( storage .history_state() @@ -598,6 +614,11 @@ mod recover_post_flush { let second = storage.recover_post_flush(1200).expect("second detect"); assert!(second.is_empty()); + assert_eq!( + generation_cuts(&storage), + [(1, 0)], + "a no-op must not add a cut" + ); assert_eq!( storage .history_state() @@ -1061,6 +1082,7 @@ mod tip_staleness { 0, "opening a missing Tip without invalidating history is not a recovery generation" ); + assert!(generation_cuts(&storage).is_empty()); let head = storage.open_state().expect("load open state"); assert!(head.is_some(), "recovery should have opened a fresh batch"); @@ -1210,6 +1232,10 @@ mod tip_staleness { 0, "the generation bump must roll back with the failed Tip reopen" ); + assert!( + generation_cuts(&storage).is_empty(), + "the pre-reopen cut must roll back too" + ); let invalidated_count: i64 = storage .conn .query_row( @@ -1306,6 +1332,15 @@ mod tip_staleness { .recover_post_flush(1200) .expect("detect and recover"); assert!(!invalidated.is_empty(), "should have invalidated batches"); + assert_eq!( + generation_cuts(&storage), + [(1, 0)], + "replacement directs must not enlarge the preserved prefix" + ); + assert_eq!( + storage.next_executed_input_count().unwrap(), + ExecutedInputCount::new(2) + ); let after = all_ordered_l2_txs(&mut storage); let direct_payloads: Vec<&[u8]> = after diff --git a/sequencer/src/storage/snapshot_dumps.rs b/sequencer/src/storage/snapshot_dumps.rs index de858c9e..41a1b140 100644 --- a/sequencer/src/storage/snapshot_dumps.rs +++ b/sequencer/src/storage/snapshot_dumps.rs @@ -353,7 +353,7 @@ fn baseline_snapshot_in(conn: &Connection) -> Result> { .optional() } -fn finalized_dump_in(conn: &Connection) -> Result> { +pub(super) fn finalized_dump_in(conn: &Connection) -> Result> { if let Some((batch_index, nonce, inclusion_block)) = latest_accepted_boundary_in(conn)? { let snapshot = snapshot_for_batch_in(conn, batch_index)?; return Ok(Some(FinalizedDump { From 62ec150f18a27697220158f1ca4d84ec4a6fee06 Mon Sep 17 00:00:00 2001 From: gcdepaula Date: Fri, 18 Sep 2026 09:09:37 -0300 Subject: [PATCH 17/29] test: cover C-host replication and recovery lifecycle --- .github/workflows/ci.yml | 6 +- Cargo.lock | 2 + bindings/c-app-engine/README.md | 20 +++ docs/plans/2026-07-coordination-tracks.md | 18 +-- .../2026-07-track3-feed-replay-design.md | 13 +- docs/protocol/c-application-binding.md | 7 +- docs/review/register.md | 13 +- justfile | 2 +- tests/e2e/Cargo.toml | 2 + tests/e2e/src/cold_replica.rs | 120 +++++++++++++----- tests/e2e/src/lib.rs | 3 + tests/e2e/src/main.rs | 26 ++-- tests/e2e/src/test_cases.rs | 93 ++++++++++++-- tests/harness/src/lib.rs | 4 +- tests/harness/src/paths.rs | 32 +++-- tests/harness/src/replay.rs | 5 +- tests/harness/src/sequencer.rs | 64 ++++++++++ 17 files changed, 334 insertions(+), 96 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5727266e..f319dbea 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -119,17 +119,17 @@ jobs: cartesi-machine-sha256-arm64: ${{ env.CARTESI_MACHINE_SHA256_ARM64 }} install-foundry: "true" - - name: Install faketime + - name: Install native test dependencies run: | sudo apt-get update - sudo apt-get install -y faketime libfaketime + sudo apt-get install -y faketime libfaketime libclang-dev - name: Build watchdog Lua deps run: | sudo apt-get install -y libcurl4-openssl-dev build-essential pkg-config just watchdog-lua-deps - - name: Run rollups E2E tests + - name: Run rollups E2E tests (Rust and C hosts) run: just test-rollups-e2e # Runs after the e2e step so the canonical machine image is already built; diff --git a/Cargo.lock b/Cargo.lock index d0ce0bdd..8c2d8d0a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3875,6 +3875,8 @@ dependencies = [ "alloy-primitives", "alloy-sol-types", "app-core", + "c-app-engine", + "c-wallet-engine", "ethereum_ssz", "futures", "libtest-mimic", diff --git a/bindings/c-app-engine/README.md b/bindings/c-app-engine/README.md index d8ffb17c..3113c8d5 100644 --- a/bindings/c-app-engine/README.md +++ b/bindings/c-app-engine/README.md @@ -107,3 +107,23 @@ notices and vouchers, rejection/no-op progress, dump round trips, independent instances, and fatal/error classification. `cargo test -p c-app-engine --lib` also checks mixed-output ordering, copying reused engine buffers, and full-width voucher values with a small ABI fixture. + +The `c_host_` scenarios in `rollups-e2e` launch the reference C host with generated +devnet genesis, delete that source before startup, and exercise ordinary execution, +clean restart, stale recovery, and checkpoint-based rebuild. Independent +`EngineApp` replicas restore HTTP archives, discard the downloaded sources, +follow backlog and live WS inputs, and check history identity across recovery. +Ordinary execution and both recovery paths also compare the host against the +canonical machine. Run them with: + +```sh +just setup +just ensure-machine-image +cargo build --locked -p c-wallet-engine --bin c-wallet-genesis -p c-wallet-sequencer --bin c-wallet-sequencer -p rollups-e2e --bin rollups-e2e +target/debug/rollups-e2e c_host_ --nocapture +``` + +`just test-rollups-e2e` includes these scenarios in CI. This covers the reference +wallet across the C ABI; private engines still need their own integration and +canonical comparison. Rebuilding from a native recovery archive does not test +the application's canonical-machine-to-native exporter. diff --git a/docs/plans/2026-07-coordination-tracks.md b/docs/plans/2026-07-coordination-tracks.md index 6158aa1d..75c8581b 100644 --- a/docs/plans/2026-07-coordination-tracks.md +++ b/docs/plans/2026-07-coordination-tracks.md @@ -19,7 +19,7 @@ freely at this stage — no backward-compatibility constraints. **Current campaign order:** 1. Merge the implemented egress API after repository review/checks. Bart can then integrate his client; adjust the API from concrete feedback without waiting for downstream completion. -2. Extend reference C-bridge coverage in this repository. Application integrators/operators own private-engine validation, the canonical-to-native exporter and recovery drill, and representative capacity measurements before production use. +2. Application integrators/operators validate the private engine, canonical-to-native exporter and recovery drill, and representative capacity before production use. Reference C-host lifecycle coverage is part of repository CI. 3. Track 5 (fee LUT) only after the log-space-fees decision. Full restore archives now support file and directory application prefixes. @@ -63,10 +63,10 @@ until the pending log-space-fees decision lands (with Bart). The [Application contract](../protocol/application-contract.md) owns execution, engine progress, and checkpoint semantics. The [C binding guide](../protocol/c-application-binding.md) -maps that contract to native engines; its reference conformance suite is -implemented. End-to-end native snapshot-to-live bootstrap remains an integration -gate, alongside the private DEX engine when available. Reference bridge -conformance cannot establish private-engine correctness. +maps that contract to native engines. Its reference conformance suite and C-host +process scenarios cover snapshot-to-live bootstrap, restart, standard recovery, +and fresh-era rebuild with canonical comparison. Reference bridge conformance +cannot establish private-engine correctness. Remaining checks need the actual consumer: @@ -79,10 +79,10 @@ Remaining checks need the actual consumer: resume metadata from canonical execution. Add the integration check to the release validation once the actual artifacts are available; no generic trait or deployment gate currently enforces this requirement. -- Exercise snapshot-to-live bootstrap and canonical comparison through the C - host in CI; its current smoke test builds and invokes `--help`. A reusable - conformance runner needs engine-supplied genesis and meaningful accepted and - rejected inputs. Compare canonical state files, not recovery-dump layouts. +- Repeat snapshot-to-live bootstrap and canonical comparison with the private + engine's genesis and meaningful accepted/rejected inputs. The reference + `c_host_` scenarios supply the lifecycle pattern; compare canonical state + files, not recovery-dump layouts. - Verify the external scheduler's ordering, fee conversion, and recovery agreement. Publish independent-port fee vectors for the [current arithmetic](../../sequencer-core/src/fee.rs); a deferred LUT is a diff --git a/docs/plans/2026-07-track3-feed-replay-design.md b/docs/plans/2026-07-track3-feed-replay-design.md index 6551f8fd..31e06fe6 100644 --- a/docs/plans/2026-07-track3-feed-replay-design.md +++ b/docs/plans/2026-07-track3-feed-replay-design.md @@ -19,7 +19,6 @@ concrete feedback. There are no live deployments requiring compatibility. | Follow-up | Owner | When it is needed | |---|---|---| -| Native reference adapter snapshot-to-live and recovery coverage | Sequencer maintainers | Additional repository conformance coverage after merge; the wallet projection tests already exercise this API's replay and recovery contract. | | Private DEX scheduler, indexer, and database backups | Bart / application integration | After merge, while adopting the API. Verify complete checkpoint/claim association and scheduler replay; report missing fields or awkward workflow for adjustment. | | Canonical-to-native export and incident rehearsal | Application integration and operators, under Track 6 | Before relying on that application's recovery procedure in production. | | Ingress latency, indexing headroom, and recovery capacity | Sequencer/application maintainers and deployment operators | Before claiming support for the target deployment workload; measure historical serving alongside ordinary traffic. | @@ -29,6 +28,14 @@ wallet's nonempty cold bootstrap, concurrent replay/live delivery, recovery and rebootstrap, canonical-machine gates, and local latency measurements. Those results do not establish private-engine conformance or target-deployment capacity. +The reference C host has process coverage in the `c_host_` rollups E2E scenarios: +generated genesis, source-independent `EngineApp` snapshot restore, concurrent +backlog/live replay, clean restart, stale recovery, and a fresh-era rebuild. +Ordinary execution and recovery compare against the canonical machine. The +[C binding guide](../../bindings/c-app-engine/README.md#reference-wallet) owns +the commands and scope. This closes the reference-host follow-up; the private +engine and canonical-to-native exporter still require their own evidence. + ## Application projections and recovery Historical bootstrap and standard-recovery checkpoint reuse are implemented. @@ -83,8 +90,8 @@ replacement directs, empty/no-op invalidation, nonzero baselines, and rollback. identifying an unsound projection checkpoint, including one below new `K`, and restoring an earlier trusted backup or genesis. The API does not certify the client's projection. -2. **Repository conformance and deployment readiness.** Extend native snapshot-to-live - validation and measure latency, historical serving cost, projection throughput, +2. **Deployment readiness.** Validate the private native engine and measure + latency, historical serving cost, projection throughput, catch-up headroom, and recovery time. Track 6 independently owns the versioned canonical-machine exporter and native-state-unavailable drill. diff --git a/docs/protocol/c-application-binding.md b/docs/protocol/c-application-binding.md index 49604648..49b03f1c 100644 --- a/docs/protocol/c-application-binding.md +++ b/docs/protocol/c-application-binding.md @@ -49,5 +49,8 @@ The same implementation may be compiled for native execution and the canonical machine. This does not establish equivalent behavior across targets: the application must preserve deterministic state and output bytes, including the checkpoint's canonical comparison file. Reference wallet ABI tests exercise -the host integration; they do not establish private DEX conformance or -equivalence between native and machine execution. +the adapter contract; the `c_host_` process scenarios exercise snapshot replication, +restart, both recovery paths, and canonical-machine comparison as described in +the [build guide](../../bindings/c-app-engine/README.md#reference-wallet). +These tests establish reference-wallet agreement for their workloads, not +private DEX conformance. diff --git a/docs/review/register.md b/docs/review/register.md index 90c9713a..238f2527 100644 --- a/docs/review/register.md +++ b/docs/review/register.md @@ -51,6 +51,13 @@ exposure in an actual deployment was established by this review. ## Bounded investigations and cleanup +- **Intermittent process-lock test failure.** The macOS workspace suite can + report `Locked` at the final reacquisition in + `dropped_runtime_scope_keeps_lock_until_detached_worker_stops` in + [`workers.rs`](../../sequencer/src/commands/run/workers.rs); an isolated rerun + passes. The worker drops its scope before signalling completion, so a simple + worker-completion race does not explain the failure. Identify any remaining + descriptor/process ownership before changing the assertion or lock behavior. - **Transient SQLite contention stops the submitter.** Read handles use a 50 ms busy timeout; a storage/open failure escapes the submitter loop. BUSY/LOCKED are nonterminal but project to unclassified exit 1, causing @@ -111,11 +118,11 @@ replay; no new snapshot lifecycle is implied. ## Integration work owned elsewhere -- [Track 3 integration gates](../plans/2026-07-track3-feed-replay-design.md): - native snapshot-to-live replication and representative deployment latency, +- [Track 3 follow-ups](../plans/2026-07-track3-feed-replay-design.md): + private-engine snapshot-to-live replication and representative deployment latency, including checkpoint creation and complete L1-reconciliation turns. - [Track 6](../plans/2026-07-coordination-tracks.md#track-6--dump--application-api-redesign): - external engine/scheduler agreement, C-host end-to-end coverage, independent + external engine/scheduler agreement, application-specific recovery export, independent fee-conversion vectors, and consumer-driven ABI/checkpoint decisions. The [2026-09-16 validation record](2026-09-16-track3-validation.md) supports diff --git a/justfile b/justfile index 91ccdfee..2b985192 100644 --- a/justfile +++ b/justfile @@ -65,7 +65,7 @@ test-sequencer: test-rollups-e2e: setup ensure-machine-image ensure-sepolia-machine-image just watchdog-lua-deps - cargo build -p wallet-sequencer --bin wallet-sequencer-devnet -p rollups-e2e --bin rollups-e2e + cargo build -p wallet-sequencer --bin wallet-sequencer-devnet -p c-wallet-engine --bin c-wallet-genesis -p c-wallet-sequencer --bin c-wallet-sequencer -p rollups-e2e --bin rollups-e2e cargo run -p rollups-e2e --bin rollups-e2e ensure-machine-image: diff --git a/tests/e2e/Cargo.toml b/tests/e2e/Cargo.toml index 8b5845ac..05c72d1a 100644 --- a/tests/e2e/Cargo.toml +++ b/tests/e2e/Cargo.toml @@ -17,6 +17,8 @@ path = "src/bin/devnet_stack.rs" rollups-harness = { path = "../harness" } tracing-subscriber = { workspace = true } app-core = { path = "../../examples/app-core" } +c-app-engine = { path = "../../bindings/c-app-engine" } +c-wallet-engine = { path = "../../examples/c-wallet-engine" } sequencer-core = { path = "../../sequencer-core" } sequencer-rust-client = { path = "../../sdk/rust-client" } alloy-primitives = { workspace = true } diff --git a/tests/e2e/src/cold_replica.rs b/tests/e2e/src/cold_replica.rs index 364888b9..97d49591 100644 --- a/tests/e2e/src/cold_replica.rs +++ b/tests/e2e/src/cold_replica.rs @@ -7,8 +7,11 @@ use std::time::Duration; use alloy_primitives::U256; +use app_core::application::WalletApp; +use rollups_harness::replay::apply_ws_message; use rollups_harness::{ManagedSequencer, ReplayWalletApp, TestSigner, WsClient}; use sequencer_core::api::WsTxMessage; +use sequencer_core::application::Application; use sequencer_rust_client::{ HistoryClaim, HistoryPolicyError, SequencerClient, SnapshotResponse, SubscribeError, }; @@ -17,12 +20,20 @@ use crate::ScenarioResult; use crate::test_cases::advance_live_frame_until_covers; pub(crate) async fn run(runtime: &mut ManagedSequencer) -> ScenarioResult<()> { - tokio::time::timeout(Duration::from_secs(120), run_scenario(runtime)) + run_with::(runtime).await +} + +pub(crate) async fn run_c(runtime: &mut ManagedSequencer) -> ScenarioResult<()> { + run_with::(runtime).await +} + +async fn run_with(runtime: &mut ManagedSequencer) -> ScenarioResult<()> { + tokio::time::timeout(Duration::from_secs(120), run_scenario::(runtime)) .await .map_err(|_| "cold replica scenario exceeded its 120-second deadline")? } -async fn run_scenario(runtime: &mut ManagedSequencer) -> ScenarioResult<()> { +async fn run_scenario(runtime: &mut ManagedSequencer) -> ScenarioResult<()> { let client = SequencerClient::new(runtime.endpoint())?; let genesis = client.latest_snapshot().await?; assert_eq!(genesis.claim.next_input.get(), 0); @@ -61,11 +72,11 @@ async fn run_scenario(runtime: &mut ManagedSequencer) -> ScenarioResult<()> { // after snapshot selection and before archive restoration or subscription. alice_l2.transfer(bob_address, U256::from(1_000)).await?; record(&mut reference_ws, &mut reference, &mut history).await?; - let (mut replica, downloaded_claim) = restore(snapshot).await?; + let (mut replica, downloaded_claim) = restore::(snapshot).await?; assert_eq!(downloaded_claim, original_claim); let snapshot_reference = replay_prefix(&history, snapshot_count)?; - assert_same_state(&replica, &snapshot_reference)?; - assert!(replica.executed_input_count() < reference.executed_input_count()); + assert_same_state(&mut replica, &snapshot_reference)?; + assert!(replica.executed_input_count().get() < reference.executed_input_count()); let backlog_deposit = alice_l1 .mint_and_deposit_supported_token(U256::from(70_000)) @@ -92,8 +103,8 @@ async fn run_scenario(runtime: &mut ManagedSequencer) -> ScenarioResult<()> { ScenarioResult::Ok(()) }; let consumer = async { - replica.apply(replica_ws.next_message().await?)?; - assert!(replica.executed_input_count() < backlog_head); + apply_ws_message(&mut replica, replica_ws.next_message().await?)?; + assert!(replica.executed_input_count().get() < backlog_head); first_replayed .send(()) .map_err(|_| "producer dropped the replay barrier")?; @@ -101,7 +112,7 @@ async fn run_scenario(runtime: &mut ManagedSequencer) -> ScenarioResult<()> { consume_until(&mut replica_ws, &mut replica, catch_up_target).await }; futures::try_join!(producer, consumer)?; - assert_same_state(&replica, &reference)?; + assert_same_state(&mut replica, &reference)?; replica_ws .expect_no_message_for(Duration::from_millis(100)) .await?; @@ -121,15 +132,47 @@ async fn run_scenario(runtime: &mut ManagedSequencer) -> ScenarioResult<()> { reference.executed_input_count(), ) .await?; - assert_same_state(&replica, &reference)?; + assert_same_state(&mut replica, &reference)?; assert!(replica.last_executed_safe_block() > clock_before_live); assert_eq!( - replica.current_user_balance(bob_address), + reference.current_user_balance(bob_address), U256::from(15_000) ); + // Preserve the replica's claim across a clean restart, then make the + // restarted host execute against its restored nonempty snapshot and suffix. + let restart_claim = HistoryClaim { + next_input: replica.executed_input_count(), + ..downloaded_claim + }; + drop(replica_ws); + drop(reference_ws); + runtime.stop().await?; + runtime.respawn().await?; + let client = SequencerClient::new(runtime.endpoint())?; + let mut replica_ws = WsClient::connect(&client, restart_claim).await?; + let mut reference_ws = WsClient::connect(&client, restart_claim).await?; + replica_ws + .expect_no_message_for(Duration::from_millis(100)) + .await?; + let mut alice_l2 = runtime.wallet_l2(TestSigner::from_default(1)?)?; + alice_l2.set_next_nonce(reference.current_user_nonce(alice_address)); + alice_l2.transfer(bob_address, U256::from(6_000)).await?; + record(&mut reference_ws, &mut reference, &mut history).await?; + consume_until( + &mut replica_ws, + &mut replica, + reference.executed_input_count(), + ) + .await?; + assert_same_state(&mut replica, &reference)?; + assert_eq!( + reference.current_user_balance(bob_address), + U256::from(21_000) + ); + let stale_claim = HistoryClaim { - next_input: sequencer_rust_client::ExecutedInputCount::new(replica.executed_input_count()), + next_input: replica.executed_input_count(), ..downloaded_claim }; drop(replica_ws); @@ -151,7 +194,7 @@ async fn run_scenario(runtime: &mut ManagedSequencer) -> ScenarioResult<()> { HistoryPolicyError::StaleGeneration { .. } )) )); - let (mut recovered, fresh_claim) = restore(client.latest_snapshot().await?).await?; + let (mut recovered, fresh_claim) = restore::(client.latest_snapshot().await?).await?; assert_eq!(fresh_claim.version.era_id, original_claim.version.era_id); assert_eq!( fresh_claim.version.recovery_generation.get(), @@ -162,7 +205,7 @@ async fn run_scenario(runtime: &mut ManagedSequencer) -> ScenarioResult<()> { // Reconstruct the expected replacement branch independently: the accepted // prefix survives, optimistic user ops disappear, and L1 directs replay. let mut recovered_reference = replay_prefix(&history, snapshot_count)?; - assert_same_state(&recovered, &recovered_reference)?; + assert_same_state(&mut recovered, &recovered_reference)?; for message in &history[snapshot_count as usize..] { if let WsTxMessage::DirectInput { .. } = message { let mut direct = message.clone(); @@ -179,19 +222,22 @@ async fn run_scenario(runtime: &mut ManagedSequencer) -> ScenarioResult<()> { recovered_reference.executed_input_count(), ) .await?; - assert_same_state(&recovered, &recovered_reference)?; - assert_eq!(recovered.current_user_balance(bob_address), U256::ZERO); - assert!(recovered.executed_input_count() < replica.executed_input_count()); + assert_same_state(&mut recovered, &recovered_reference)?; + assert_eq!( + recovered_reference.current_user_balance(bob_address), + U256::ZERO + ); + assert!(recovered.executed_input_count().get() < replica.executed_input_count().get()); let mut alice_l2 = runtime.wallet_l2(TestSigner::from_default(1)?)?; alice_l2.set_next_nonce(recovered_reference.current_user_nonce(alice_address)); alice_l2.transfer(bob_address, U256::from(6_000)).await?; let resumed = recovered_ws.expect_user_op_from(alice_address).await?; - recovered.apply(resumed.clone())?; + apply_ws_message(&mut recovered, resumed.clone())?; recovered_reference.apply(resumed)?; - assert_same_state(&recovered, &recovered_reference)?; + assert_same_state(&mut recovered, &recovered_reference)?; assert_eq!( - recovered.current_user_balance(bob_address), + recovered_reference.current_user_balance(bob_address), U256::from(6_000) ); recovered_ws @@ -200,7 +246,9 @@ async fn run_scenario(runtime: &mut ManagedSequencer) -> ScenarioResult<()> { Ok(()) } -async fn restore(snapshot: SnapshotResponse) -> ScenarioResult<(ReplayWalletApp, HistoryClaim)> { +pub(crate) async fn restore( + snapshot: SnapshotResponse, +) -> ScenarioResult<(A, HistoryClaim)> { let claim = snapshot.claim; assert_eq!( snapshot.response.headers()["Content-Type"], @@ -210,8 +258,8 @@ async fn restore(snapshot: SnapshotResponse) -> ScenarioResult<(ReplayWalletApp, let directory = tempfile::tempdir()?; tar::Archive::new(archive.as_ref()).unpack(directory.path())?; assert!(directory.path().join("info.toml").is_file()); - let app = ReplayWalletApp::from_dump(&directory.path().join("state"))?; - assert_eq!(app.executed_input_count(), claim.next_input.get()); + let app = A::from_dump(&directory.path().join("state"))?; + assert_eq!(app.executed_input_count(), claim.next_input); // Subsequent replay also checks that restoring does not retain a dependency // on the downloaded source directory. directory.close()?; @@ -229,15 +277,15 @@ async fn record( Ok(()) } -async fn consume_until( +async fn consume_until( ws: &mut WsClient, - app: &mut ReplayWalletApp, + app: &mut A, target: u64, ) -> ScenarioResult<()> { - while app.executed_input_count() < target { - app.apply(ws.next_message().await?)?; + while app.executed_input_count().get() < target { + apply_ws_message(app, ws.next_message().await?)?; } - assert_eq!(app.executed_input_count(), target); + assert_eq!(app.executed_input_count().get(), target); Ok(()) } @@ -249,17 +297,29 @@ fn replay_prefix(history: &[WsTxMessage], count: u64) -> ScenarioResult ScenarioResult<()> { +pub(crate) fn assert_same_state( + actual: &mut A, + expected: &ReplayWalletApp, +) -> ScenarioResult<()> { assert_eq!( - actual.executed_input_count(), + actual.executed_input_count().get(), expected.executed_input_count() ); assert_eq!( actual.last_executed_safe_block(), expected.last_executed_safe_block() ); + let progress = actual.progress(); + let directory = tempfile::tempdir()?; + let checkpoint = directory.path().join("checkpoint"); + actual.create_dump(&checkpoint)?; + assert_eq!( + actual.progress(), + progress, + "checkpoint creation preserves state" + ); assert_eq!( - actual.canonical_snapshot_bytes()?, + std::fs::read(A::state_file_in_dump(&checkpoint))?, expected.canonical_snapshot_bytes()?, "all wallet state, including balances, nonces, config, count, and clock" ); diff --git a/tests/e2e/src/lib.rs b/tests/e2e/src/lib.rs index 96ef4a2b..8a9889eb 100644 --- a/tests/e2e/src/lib.rs +++ b/tests/e2e/src/lib.rs @@ -5,6 +5,9 @@ mod cold_replica; pub mod test_cases; mod watchdog_compare; +// Link the reference engine's C symbols for the EngineApp replica scenarios. +use c_wallet_engine as _; + use std::future::Future; use std::pin::Pin; diff --git a/tests/e2e/src/main.rs b/tests/e2e/src/main.rs index a2c6130c..2e30c8cd 100644 --- a/tests/e2e/src/main.rs +++ b/tests/e2e/src/main.rs @@ -4,7 +4,7 @@ use libtest_mimic::{Arguments, Trial}; use rollups_e2e::run_trial; use rollups_harness::{ - ManagedSequencer, default_devnet_sequencer_config, devnet_sequencer_config_no_faketime, + ManagedSequencer, default_c_wallet_sequencer_config, default_devnet_sequencer_config, }; fn main() { @@ -16,20 +16,22 @@ fn main() { .map(|(name, scenario)| { Trial::test(name, move || { let log_prefix = format!("rollups-e2e-{name}"); - let spawn_config = if name == "watchdog_genesis_compare_test" - || name == "deposit_transfer_withdrawal_test" - || name == "watchdog_non_genesis_divergence_test" - { - devnet_sequencer_config_no_faketime(log_prefix) - } else if name == "fixed_fee_oracle_sets_frame_fee_test" { - let mut config = default_devnet_sequencer_config(log_prefix); - // 100 → recommended fee 1456, under the wallet client's - // DEFAULT_MAX_FEE (2500) so transfers still admit. - config.fee_oracle_fixed_log_gas_price = Some(100); - config + let mut spawn_config = if name.starts_with("c_host_") { + default_c_wallet_sequencer_config(log_prefix) } else { default_devnet_sequencer_config(log_prefix) }; + let scenario_name = name.strip_prefix("c_host_").unwrap_or(name); + if scenario_name == "watchdog_genesis_compare_test" + || scenario_name == "deposit_transfer_withdrawal_test" + || scenario_name == "watchdog_non_genesis_divergence_test" + { + spawn_config.faketime = false; + } else if scenario_name == "fixed_fee_oracle_sets_frame_fee_test" { + // 100 → recommended fee 1456, under the wallet client's + // DEFAULT_MAX_FEE (2500) so transfers still admit. + spawn_config.fee_oracle_fixed_log_gas_price = Some(100); + } run_trial(name, || async move { let mut runtime = ManagedSequencer::spawn(spawn_config).await?; let scenario_result = scenario(&mut runtime).await; diff --git a/tests/e2e/src/test_cases.rs b/tests/e2e/src/test_cases.rs index e04e86e2..d2c8448e 100644 --- a/tests/e2e/src/test_cases.rs +++ b/tests/e2e/src/test_cases.rs @@ -11,9 +11,10 @@ use rollups_harness::{ WsClient, sign_user_op_hex, }; use sequencer_core::api::{TxRequest, WsTxMessage}; +use sequencer_core::application::Application; use sequencer_core::fee::fee_to_linear; use sequencer_core::user_op::UserOp; -use sequencer_rust_client::SequencerClient; +use sequencer_rust_client::{HistoryPolicyError, SequencerClient, SubscribeError}; const NO_WS_MESSAGE_WAIT: Duration = Duration::from_secs(1); @@ -152,6 +153,22 @@ struct ExpectedWalletState { pub fn test_cases() -> Vec<(&'static str, ScenarioFn)> { vec![ + ( + "c_host_cold_replica_snapshot_backlog_live_recovery_test", + |runtime| Box::pin(crate::cold_replica::run_c(runtime)), + ), + ("c_host_setup_recovery_round_trip_test", |runtime| { + Box::pin(run_setup_recovery_round_trip_test::(runtime)) + }), + ("c_host_deposit_transfer_withdrawal_test", |runtime| { + Box::pin(run_deposit_transfer_withdrawal_test(runtime)) + }), + ("c_host_recovery_after_stale_batches_test", |runtime| { + Box::pin(run_recovery_after_stale_batches_test(runtime)) + }), + ("c_host_restart_and_replay_test", |runtime| { + Box::pin(run_restart_and_replay_test(runtime)) + }), ( "cold_replica_snapshot_backlog_live_recovery_test", |runtime| Box::pin(crate::cold_replica::run(runtime)), @@ -197,7 +214,9 @@ pub fn test_cases() -> Vec<(&'static str, ScenarioFn)> { Box::pin(run_recovery_after_stale_batches_test(runtime)) }), ("setup_recovery_round_trip_test", |runtime| { - Box::pin(run_setup_recovery_round_trip_test(runtime)) + Box::pin(run_setup_recovery_round_trip_test::< + app_core::application::WalletApp, + >(runtime)) }), ("sequencer_outage_pre_danger_no_recovery_test", |runtime| { Box::pin(run_sequencer_outage_pre_danger_no_recovery_test(runtime)) @@ -878,6 +897,33 @@ async fn run_restart_and_replay_test(runtime: &mut ManagedSequencer) -> Scenario replay_before_restart.last_executed_safe_block(), "mirror safe-block clock must match the pre-restart live replay clock", ); + + // Reading the persisted feed alone does not prove the restarted engine + // restored its balances and nonce. Require a fresh execution at nonce 1. + let mut resumed_alice = runtime.wallet_l2(alice)?; + resumed_alice.set_next_nonce(1); + resumed_alice.transfer(bob_address, U256::from(1)).await?; + let resumed = ws_after_restart.expect_user_op_from(alice_address).await?; + replay_after_restart.apply(resumed.clone())?; + replay_before_restart.apply(resumed)?; + assert_eq!( + replay_after_restart.canonical_snapshot_bytes()?, + replay_before_restart.canonical_snapshot_bytes()? + ); + assert_wallet_state( + &replay_after_restart, + ExpectedWalletState { + address: alice_address, + balance: expected_alice - U256::from(1) - gas, + nonce: 2, + }, + ExpectedWalletState { + address: bob_address, + balance: expected_bob + U256::from(1), + nonce: 1, + }, + 4, + ); Ok(()) } @@ -1368,7 +1414,9 @@ async fn drive_promotion_and_capture( runtime.capture_finalized_checkpoint().await } -async fn run_setup_recovery_round_trip_test(runtime: &mut ManagedSequencer) -> ScenarioResult<()> { +async fn run_setup_recovery_round_trip_test( + runtime: &mut ManagedSequencer, +) -> ScenarioResult<()> { runtime.set_max_batch_open_seconds(Some(5)); runtime.restart().await?; @@ -1399,6 +1447,10 @@ async fn run_setup_recovery_round_trip_test(runtime: &mut ManagedSequencer) -> S // Drive the batch to seal + be accepted + promoted, and capture the // resulting finalized snapshot as the recovery checkpoint. let checkpoint = drive_promotion_and_capture(runtime).await?; + let old_claim = SequencerClient::new(runtime.endpoint())? + .latest_snapshot() + .await? + .claim; eprintln!( "recovery checkpoint: B={} N={}", checkpoint.checkpoint_block, checkpoint.resume_nonce @@ -1417,19 +1469,33 @@ async fn run_setup_recovery_round_trip_test(runtime: &mut ManagedSequencer) -> S runtime.set_mine_l1_during_boot(true); runtime.respawn().await?; - // Recovery booted (the respawn above succeeded — `setup --recovery` rebuilt - // the DB and `run` started clean). Now prove the fold preserved Alice's - // logical state: a transfer at her *continuing* nonce (1) is accepted. The - // sequencer validates it against the recovered state S' — a lost nonce would - // be rejected (wrong nonce), a lost balance rejected (insufficient funds). - // So acceptance is the end-to-end proof that the checkpoint's balances + - // nonces were folded into the rebuilt DB. (The local `replay` can't verify - // S' directly — the wiped pre-recovery transfer isn't re-fed — so the - // sequencer's own acceptance is the authority here.) + // A rebuilt database cannot authorize the old replica, even when its + // numerical generation/count match. Restore and subscribe in the new era. + let client = SequencerClient::new(runtime.endpoint())?; + assert!(matches!( + client.subscribe(old_claim).await, + Err(SubscribeError::History( + HistoryPolicyError::EraChanged { .. } + )) + )); + let (mut restored, fresh_claim) = + crate::cold_replica::restore::(client.latest_snapshot().await?).await?; + assert_ne!(fresh_claim.version.era_id, old_claim.version.era_id); + assert_eq!(fresh_claim.version.recovery_generation.get(), 0); + assert!(fresh_claim.next_input.get() > 0); + crate::cold_replica::assert_same_state(&mut restored, &replay)?; + let mut resumed_ws = WsClient::connect(&client, fresh_claim).await?; + + // A continuing nonce exercises the recovered host's state as well as the + // independently restored replica and the retained reference history. let mut alice_l2_after = runtime.wallet_l2(alice)?; alice_l2_after.set_next_nonce(1); let post_transfer = U256::from(70_000_u64); alice_l2_after.transfer(bob_address, post_transfer).await?; + let message = resumed_ws.expect_user_op_from(alice_address).await?; + rollups_harness::replay::apply_ws_message(&mut restored, message.clone())?; + replay.apply(message)?; + crate::cold_replica::assert_same_state(&mut restored, &replay)?; // Explicit recovery-correctness assertions, beyond the structural // `assert_schema_invariants` (which checks `0..`-from-anchor contiguity): @@ -1458,8 +1524,7 @@ async fn run_setup_recovery_round_trip_test(runtime: &mut ManagedSequencer) -> S // from genesis — the pre-wipe batches (nonce < N') plus the post-recovery // batches at the resume nonce N' — so agreement proves the fold-rebuilt state // S' and the resumed submission both land exactly on the canonical chain - // state. The local `replay` can't check this (the wiped history is never - // re-fed), so the watchdog's independent CM is the authority. + // state. This checks native execution against the independent CM target. let floor_inclusion_block = runtime.finalized_inclusion_block().await?.unwrap_or(0); let batches_before = runtime.count_batches()?; for _ in 0..TRANSFERS_TO_FORCE_BATCH_CLOSE { diff --git a/tests/harness/src/lib.rs b/tests/harness/src/lib.rs index 53c31619..b43f1aa1 100644 --- a/tests/harness/src/lib.rs +++ b/tests/harness/src/lib.rs @@ -18,8 +18,8 @@ pub use rollups::{DEVNET_CHAIN_ID, DevnetRollupsStack}; pub use sequencer::{ BatchCounts, DEFAULT_DEVNET_SEQUENCER_BIN, DEFAULT_TEST_LOGS_DIR, ManagedSequencer, ManagedSequencerConfig, RecoveryCheckpoint, RecoverySetupParams, RespawnAttemptOutcome, - RespawnPolicy, StackChildExit, default_devnet_sequencer_config, - devnet_sequencer_config_no_faketime, + RespawnPolicy, StackChildExit, default_c_wallet_sequencer_config, + default_devnet_sequencer_config, devnet_sequencer_config_no_faketime, }; pub use wallet::{ TestSigner, WalletL1Client, WalletL2Client, address_from_signing_key, sign_user_op_hex, diff --git a/tests/harness/src/paths.rs b/tests/harness/src/paths.rs index 65c8dc08..b0573f11 100644 --- a/tests/harness/src/paths.rs +++ b/tests/harness/src/paths.rs @@ -39,29 +39,35 @@ pub fn devnet_machine_image_path() -> PathBuf { workspace_root().join(DEFAULT_DEVNET_MACHINE_IMAGE_PATH) } -const DEVNET_SEQUENCER_BIN: &str = "wallet-sequencer-devnet"; - /// Resolve the `wallet-sequencer-devnet` binary built for the current Cargo invocation. -/// -/// Prefers `CARGO_TARGET_DIR` (set by `cargo run` / `cargo test` in sandboxes and -/// custom target dirs) over the workspace `target/debug/` tree, which may be stale -/// when builds only run through Cargo with a redirected target directory. pub fn resolve_devnet_sequencer_bin() -> PathBuf { - if let Ok(path) = std::env::var("CARGO_BIN_EXE_WALLET_SEQUENCER_DEVNET") { + resolve_debug_bin( + "wallet-sequencer-devnet", + "CARGO_BIN_EXE_WALLET_SEQUENCER_DEVNET", + ) +} + +pub fn resolve_c_wallet_sequencer_bin() -> PathBuf { + resolve_debug_bin("c-wallet-sequencer", "CARGO_BIN_EXE_C_WALLET_SEQUENCER") +} + +pub fn resolve_c_wallet_genesis_bin() -> PathBuf { + resolve_debug_bin("c-wallet-genesis", "CARGO_BIN_EXE_C_WALLET_GENESIS") +} + +// Prefer the active Cargo target over a potentially stale workspace target/debug. +fn resolve_debug_bin(binary: &str, override_env: &str) -> PathBuf { + if let Ok(path) = std::env::var(override_env) { let path = PathBuf::from(path); if path.exists() { return path; } } if let Ok(target) = std::env::var("CARGO_TARGET_DIR") { - let path = PathBuf::from(target) - .join("debug") - .join(DEVNET_SEQUENCER_BIN); + let path = PathBuf::from(target).join("debug").join(binary); if path.exists() { return path; } } - workspace_root() - .join("target/debug") - .join(DEVNET_SEQUENCER_BIN) + workspace_root().join("target/debug").join(binary) } diff --git a/tests/harness/src/replay.rs b/tests/harness/src/replay.rs index 4b1bf678..e715bab8 100644 --- a/tests/harness/src/replay.rs +++ b/tests/harness/src/replay.rs @@ -63,10 +63,7 @@ impl ReplayWalletApp { } } -pub(crate) fn apply_ws_message( - app: &mut A, - message: WsTxMessage, -) -> HarnessResult<()> { +pub fn apply_ws_message(app: &mut A, message: WsTxMessage) -> HarnessResult<()> { let expected = app.executed_input_count().get(); if message.offset() != expected { return Err(std::io::Error::other(format!( diff --git a/tests/harness/src/sequencer.rs b/tests/harness/src/sequencer.rs index 0ab10434..cb5fc629 100644 --- a/tests/harness/src/sequencer.rs +++ b/tests/harness/src/sequencer.rs @@ -45,6 +45,9 @@ pub const DEFAULT_TEST_LOGS_DIR: &str = "tests/e2e/results"; #[derive(Debug, Clone)] pub struct ManagedSequencerConfig { pub sequencer_bin: PathBuf, + /// Optional genesis tool invoked as ` devnet` for + /// initial setup. Its output is deleted before `run` and never regenerated. + pub genesis_bin: Option, pub log_prefix: String, pub logs_dir: PathBuf, /// When false, the child runs without libfaketime (for tests that never @@ -178,6 +181,7 @@ pub struct ManagedSequencer { pub fn default_devnet_sequencer_config(log_prefix: impl Into) -> ManagedSequencerConfig { ManagedSequencerConfig { sequencer_bin: paths::resolve_devnet_sequencer_bin(), + genesis_bin: None, log_prefix: log_prefix.into(), logs_dir: PathBuf::from(DEFAULT_TEST_LOGS_DIR), faketime: true, @@ -185,6 +189,14 @@ pub fn default_devnet_sequencer_config(log_prefix: impl Into) -> Managed } } +pub fn default_c_wallet_sequencer_config(log_prefix: impl Into) -> ManagedSequencerConfig { + ManagedSequencerConfig { + sequencer_bin: paths::resolve_c_wallet_sequencer_bin(), + genesis_bin: Some(paths::resolve_c_wallet_genesis_bin()), + ..default_devnet_sequencer_config(log_prefix) + } +} + /// Devnet config without libfaketime (watchdog compare and other wall-clock-neutral tests). pub fn devnet_sequencer_config_no_faketime( log_prefix: impl Into, @@ -203,6 +215,10 @@ impl ManagedSequencer { } else { paths::resolve_from_workspace(&config.sequencer_bin) }; + let genesis_bin = config + .genesis_bin + .as_ref() + .map(paths::resolve_from_workspace); let log_prefix = config.log_prefix; let rollups = DevnetRollupsStack::spawn(log_prefix.as_str(), logs_dir.as_path()).await?; @@ -243,6 +259,7 @@ impl ManagedSequencer { // Default batch-open deadline on first boot. None, config.fee_oracle_fixed_log_gas_price, + genesis_bin.as_deref(), ) .await?; @@ -1102,6 +1119,7 @@ impl ManagedSequencer { self.recovery_setup.as_ref(), self.max_batch_open_seconds, self.fee_oracle_fixed_log_gas_price, + None, ) .await?; self.child = child; @@ -1249,6 +1267,7 @@ async fn spawn_sequencer_process( recovery: Option<&RecoverySetupParams>, max_batch_open_seconds: Option, fee_oracle_fixed_log_gas_price: Option, + genesis_bin: Option<&Path>, ) -> HarnessResult { let (endpoint, http_addr) = build_local_endpoint()?; let log_path = timestamped_log_path(logs_dir, log_prefix); @@ -1287,6 +1306,41 @@ async fn spawn_sequencer_process( let chain_id = chain_id_override.unwrap_or(DEVNET_CHAIN_ID); let bin = path_as_str(sequencer_bin)?.to_owned(); + let genesis_dir = if let Some(genesis_bin) = genesis_bin { + let dir = TempDir::new()?; + let mut command = Command::new(genesis_bin); + command + .kill_on_drop(true) + .arg(dir.path().join("state")) + .arg("devnet"); + let output = tokio::time::timeout(DEFAULT_SEQUENCER_START_TIMEOUT, command.output()) + .await + .map_err(|_| { + io_other(format!( + "genesis tool '{}' timed out", + genesis_bin.display() + )) + })? + .map_err(|err| { + io_other(format!( + "failed to run genesis tool '{}': {err}", + genesis_bin.display() + )) + })?; + if !output.status.success() { + return Err(io_other(format!( + "genesis tool '{}' failed: status={}: {}", + genesis_bin.display(), + output.status, + String::from_utf8_lossy(&output.stderr) + )) + .into()); + } + Some(dir) + } else { + None + }; + // libfaketime is applied via env vars (not the `faketime` wrapper binary), // which the file-based FAKETIME_TIMESTAMP_FILE mechanism reads on every // time call (FAKETIME_NO_CACHE=1) so tests can shift the clock at runtime. @@ -1305,6 +1359,10 @@ async fn spawn_sequencer_process( // (e.g. a chain-id mismatch) surfaces here as a non-zero exit, just as it // surfaced from the monolithic boot before the split. let mut setup_cmd = Command::new(&bin); + setup_cmd.env_remove("CARTESI_SEQUENCER_STATE_FILE"); + if let Some(dir) = &genesis_dir { + setup_cmd.arg("--state-file").arg(dir.path().join("state")); + } if let (Some(lib), Some(rc)) = (libfaketime_path, faketime_rc_path) { apply_faketime_env(&mut setup_cmd, lib, rc)?; } @@ -1396,10 +1454,16 @@ async fn spawn_sequencer_process( )) .into()); } + if let Some(dir) = genesis_dir { + // Neither startup nor later recovery may depend on the original source. + dir.close() + .map_err(|err| io_other(format!("remove initial genesis after setup: {err}")))?; + } // Phase B — `run` (re-spawned on every restart; reads identity from the // setup DB). let mut run_cmd = Command::new(&bin); + run_cmd.env_remove("CARTESI_SEQUENCER_STATE_FILE"); if let (Some(lib), Some(rc)) = (libfaketime_path, faketime_rc_path) { apply_faketime_env(&mut run_cmd, lib, rc)?; } From 422dae48ebeed2631f2d3098f2cefba386e1b484 Mon Sep 17 00:00:00 2001 From: gcdepaula Date: Fri, 18 Sep 2026 20:14:25 -0300 Subject: [PATCH 18/29] fix: refuse finalized checkpoint exports after divergence --- README.md | 6 +- sequencer/src/egress/api/snapshot.rs | 25 ++++- .../integration_tests/snapshot_endpoints.rs | 97 +++++++++++++++++++ sequencer/src/storage/egress/historical.rs | 17 ++-- .../src/storage/egress/historical/tests.rs | 8 +- sequencer/src/storage/mod.rs | 2 +- sequencer/src/storage/snapshot_dumps.rs | 39 ++++++-- 7 files changed, 166 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index 515989c9..1e791634 100644 --- a/README.md +++ b/README.md @@ -406,8 +406,10 @@ and `X-Executed-Input-Count`, selected atomically with the artifact lease. Streaming holds the lease until the response ends or the client disconnects. The accepted endpoints return `404` until a comparable checkpoint exists: genesis is comparable at block zero; a rebuilt baseline is restorable but only -a later accepted batch establishes a comparison point. Divergence blocks -publication of the accepted checkpoint. See [snapshot lifecycle](docs/snapshots/lifecycle.md). +a later accepted batch establishes a comparison point. Known divergence makes +all three finalized endpoints return `503 UNAVAILABLE`, including conditional +state requests. The check shares the checkpoint-selection transaction, before +any lease or archive is created. See [snapshot lifecycle](docs/snapshots/lifecycle.md). ## Storage Model diff --git a/sequencer/src/egress/api/snapshot.rs b/sequencer/src/egress/api/snapshot.rs index 1c83041a..f5e98c8f 100644 --- a/sequencer/src/egress/api/snapshot.rs +++ b/sequencer/src/egress/api/snapshot.rs @@ -23,9 +23,11 @@ use tokio::fs::File; use tokio::io::{AsyncRead, ReadBuf}; use tokio_util::io::{ReaderStream, SyncIoBridge}; -use crate::http::{StorageTaskError, storage_task}; +use crate::http::{ApiError, StorageTaskError, storage_task}; use crate::runtime::shutdown::{RuntimeScope, abort_terminal}; -use crate::storage::{FinalizedLease, LeaseGuard, LeasedDump, ReleaseScheduler, Storage}; +use crate::storage::{ + FinalizedLease, FinalizedSelectionError, LeaseGuard, LeasedDump, ReleaseScheduler, Storage, +}; type BoxError = StorageTaskError; @@ -89,7 +91,7 @@ async fn finalized_inclusion_block(State(state): State>) - }) .into_response(), Ok(None) => StatusCode::NOT_FOUND.into_response(), - Err(err) => internal_error("read finalized inclusion block", err), + Err(err) => finalized_error("read finalized inclusion block", err), } } @@ -105,7 +107,7 @@ async fn finalized_state( } = match acquire_finalized(&state).await { Ok(Some(leased)) => leased, Ok(None) => return StatusCode::NOT_FOUND.into_response(), - Err(err) => return internal_error("acquire finalized lease", err), + Err(err) => return finalized_error("acquire finalized lease", err), }; let etag = format!("\"block-{inclusion_block}\""); @@ -170,7 +172,7 @@ async fn finalized_snapshot(State(state): State>) -> Respo archive_response(&state, dump, Some(checkpoint)) } Ok(None) => StatusCode::NOT_FOUND.into_response(), - Err(err) => internal_error("acquire accepted snapshot lease", err), + Err(err) => finalized_error("acquire accepted snapshot lease", err), } } @@ -368,6 +370,19 @@ fn internal_error(context: &str, err: impl std::fmt::Display) -> Response { StatusCode::INTERNAL_SERVER_ERROR.into_response() } +fn finalized_error(context: &str, err: StorageTaskError) -> Response { + if matches!( + err.downcast_ref::(), + Some(FinalizedSelectionError::CanonicalDivergence) + ) { + return ApiError::unavailable( + "canonical divergence prevents accepted checkpoint selection", + ) + .into_response(); + } + internal_error(context, err) +} + #[cfg(test)] mod tests { use super::*; diff --git a/sequencer/src/integration_tests/snapshot_endpoints.rs b/sequencer/src/integration_tests/snapshot_endpoints.rs index 11b0eb78..05d3c659 100644 --- a/sequencer/src/integration_tests/snapshot_endpoints.rs +++ b/sequencer/src/integration_tests/snapshot_endpoints.rs @@ -180,6 +180,103 @@ fn archive_state(bytes: &[u8]) -> Vec { .unwrap() } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn same_block_divergence_refuses_every_finalized_endpoint() { + use crate::storage::test_helpers::{ + default_protocol_timing, local_batch_payload, pin_test_deployment_identity, + }; + use crate::storage::{ExecutedInputCount, SafeInputRange, StoredSafeInput}; + use sequencer_core::batch::{Batch, Frame}; + use sequencer_core::scheduler::{Scheduler, SchedulerConfig, SchedulerInput}; + use ssz::Encode; + + let db = temp_db("same-block-divergence-http"); + let root = tempfile::tempdir().unwrap(); + let submitter = alloy_primitives::Address::repeat_byte(1); + let mut storage = Storage::open(&db.path).unwrap(); + pin_test_deployment_identity(&mut storage, submitter); + let mut head = storage + .initialize_open_state(5, SafeInputRange::empty_at(0)) + .unwrap(); + let prefix = write_state(root.path(), "local", b"local checkpoint", 1); + storage + .close_frame_and_batch_with_snapshot(&mut head, 5, &prefix, 0, ExecutedInputCount::ZERO) + .unwrap(); + let snapshot = storage.latest_snapshot().unwrap().unwrap(); + let inputs = vec![ + StoredSafeInput { + sender: alloy_primitives::Address::repeat_byte(2), + payload: vec![42], + block_number: 6, + }, + StoredSafeInput { + sender: submitter, + payload: local_batch_payload(&mut storage, 0), + block_number: 10, + }, + StoredSafeInput { + sender: submitter, + payload: Batch { + nonce: 1, + frames: vec![Frame { + safe_block: 6, + fee_price: 0, + user_ops: vec![], + }], + } + .as_ssz_bytes(), + block_number: 10, + }, + ]; + storage + .append_safe_inputs(10, &inputs, submitter, &default_protocol_timing()) + .unwrap(); + assert!(storage.canonical_divergence().unwrap().is_some()); + + // The foreign batch consumes the next nonce and drains a direct that the + // matching local batch did not: its snapshot is not the end of block 10. + let mut canonical = Scheduler::new(WalletApp::default(), SchedulerConfig::new(submitter)); + for input in inputs { + canonical + .process_input(SchedulerInput { + sender: input.sender, + payload: input.payload, + inclusion_block: input.block_number, + domain: sequencer_core::build_input_domain(1, alloy_primitives::Address::ZERO), + }) + .unwrap(); + } + let (app, nonce) = canonical.finish(); + assert_eq!(nonce, 2); + assert_eq!(app.executed_input_count().get(), 1); + assert_eq!(snapshot.executed_input_count, ExecutedInputCount::ZERO); + + let server = start_server(&db.path).await.expect("test listener"); + let client = reqwest::Client::new(); + for (path, conditional) in [ + ("/finalized_state/inclusion_block", false), + ("/finalized_state", false), + ("/finalized_state", true), + ("/finalized_snapshot", false), + ] { + let mut request = client.get(server.url(path)); + if conditional { + request = request.header("If-None-Match", "\"block-10\""); + } + let response = request.send().await.unwrap(); + assert_eq!( + response.status(), + reqwest::StatusCode::SERVICE_UNAVAILABLE, + "{path}, conditional={conditional}" + ); + assert_eq!( + response.json::().await.unwrap()["code"], + "UNAVAILABLE" + ); + assert_eq!(storage.dump_lease_count(snapshot.dump.id).unwrap(), Some(0)); + } +} + /// Transient WAL lock contention vs. a real read failure. /// /// SQLite readers can see `SQLITE_BUSY` in WAL mode during last-connection diff --git a/sequencer/src/storage/egress/historical.rs b/sequencer/src/storage/egress/historical.rs index 730879a0..2b5ba5f6 100644 --- a/sequencer/src/storage/egress/historical.rs +++ b/sequencer/src/storage/egress/historical.rs @@ -21,8 +21,9 @@ use crate::storage::history::{ }; use crate::storage::l1_inputs::query_deployment_identity; use crate::storage::mutations::batch_tree_anchor_in; -use crate::storage::safe_accepted_batches::canonical_divergence_in; -use crate::storage::snapshot_dumps::{finalized_dump_in, has_rollback_safe_snapshot_in}; +use crate::storage::snapshot_dumps::{ + FinalizedSelectionError, finalized_dump_in, has_rollback_safe_snapshot_in, +}; #[derive(Debug, thiserror::Error)] pub(crate) enum HistoricalReadError { @@ -30,8 +31,8 @@ pub(crate) enum HistoricalReadError { Policy(#[from] HistoryPolicyError), #[error("{0}")] BadRequest(String), - #[error("canonical divergence prevents accepted checkpoint selection")] - CanonicalDivergence, + #[error(transparent)] + Checkpoint(#[from] FinalizedSelectionError), #[error("reading historical L1 inputs: {0}")] Storage(#[from] rusqlite::Error), } @@ -60,12 +61,12 @@ impl Storage { "from_generation exceeds the current recovery generation".to_owned(), ))); } - if canonical_divergence_in(tx)?.is_some() { - return Ok(Err(HistoricalReadError::CanonicalDivergence)); - } + let accepted = match finalized_dump_in(tx) { + Ok(accepted) => accepted, + Err(error) => return Ok(Err(error.into())), + }; let deployment = query_deployment_identity(tx)?.ok_or(rusqlite::Error::QueryReturnedNoRows)?; - let accepted = finalized_dump_in(tx)?; if accepted.is_none() { assert!( has_rollback_safe_snapshot_in(tx)?, diff --git a/sequencer/src/storage/egress/historical/tests.rs b/sequencer/src/storage/egress/historical/tests.rs index cf1587d7..eef4d5f9 100644 --- a/sequencer/src/storage/egress/historical/tests.rs +++ b/sequencer/src/storage/egress/historical/tests.rs @@ -366,9 +366,9 @@ fn accepted_checkpoint_uses_the_exact_latest_snapshot_and_preserves_baseline() { .unwrap(); assert!(matches!( storage.history_info(None, None), - Err(HistoricalReadError::Storage( + Err(HistoricalReadError::Checkpoint(FinalizedSelectionError::Storage( rusqlite::Error::QueryReturnedNoRows - )) + ))) )); } @@ -385,7 +385,9 @@ fn canonical_divergence_cannot_be_advertised_as_an_accepted_receipt() { .unwrap(); assert!(matches!( storage.history_info(None, None), - Err(HistoricalReadError::CanonicalDivergence) + Err(HistoricalReadError::Checkpoint( + FinalizedSelectionError::CanonicalDivergence + )) )); } diff --git a/sequencer/src/storage/mod.rs b/sequencer/src/storage/mod.rs index d7fa5332..97d6fac3 100644 --- a/sequencer/src/storage/mod.rs +++ b/sequencer/src/storage/mod.rs @@ -62,7 +62,7 @@ pub use recovery::DangerStatus; pub(crate) use recovery::{RecoveryInspection, RecoveryMutationError}; pub use sequencer_core::history::{EraId, ExecutedInputCount, HistoryVersion, RecoveryGeneration}; pub use snapshot_dumps::{ - DumpRow, FinalizedDump, FinalizedLease, LeaseGuard, LeasedDump, + DumpRow, FinalizedDump, FinalizedLease, FinalizedSelectionError, LeaseGuard, LeasedDump, PersistentReleaseFailureReporter, ReleaseScheduler, Snapshot, }; diff --git a/sequencer/src/storage/snapshot_dumps.rs b/sequencer/src/storage/snapshot_dumps.rs index 41a1b140..9a5f6d2c 100644 --- a/sequencer/src/storage/snapshot_dumps.rs +++ b/sequencer/src/storage/snapshot_dumps.rs @@ -35,6 +35,14 @@ pub struct FinalizedDump { pub executed_input_count: ExecutedInputCount, } +#[derive(Debug, thiserror::Error)] +pub enum FinalizedSelectionError { + #[error("canonical divergence prevents accepted checkpoint selection")] + CanonicalDivergence, + #[error("selecting finalized checkpoint: {0}")] + Storage(#[from] rusqlite::Error), +} + pub type ReleaseScheduler = Arc) + Send + Sync + 'static>; pub type PersistentReleaseFailureReporter = Arc; @@ -153,8 +161,10 @@ impl Storage { /// The latest accepted batch must have its own snapshot. A missing artifact /// is corruption, never a request to use an older accepted snapshot. - pub fn finalized_dump(&mut self) -> Result> { - self.read(|tx| finalized_dump_in(tx)) + pub fn finalized_dump( + &mut self, + ) -> std::result::Result, FinalizedSelectionError> { + self.read(|tx| Ok(finalized_dump_in(tx)))? } pub fn latest_snapshot(&mut self) -> Result> { @@ -173,15 +183,17 @@ impl Storage { &mut self, schedule: ReleaseScheduler, report_persistent_failure: PersistentReleaseFailureReporter, - ) -> Result> { + ) -> std::result::Result, FinalizedSelectionError> { let acquired = self.write(|tx| { - let Some(snapshot) = finalized_dump_in(tx)? else { - return Ok(None); + let snapshot = match finalized_dump_in(tx) { + Ok(Some(snapshot)) => snapshot, + Ok(None) => return Ok(Ok(None)), + Err(error) => return Ok(Err(error)), }; let history_version = query_history_state(tx)?.version; acquire_dump_lease_in(tx, snapshot.dump.id)?; - Ok(Some((snapshot, history_version))) - })?; + Ok(Ok(Some((snapshot, history_version)))) + })??; Ok(acquired.map(|(snapshot, history_version)| FinalizedLease { inclusion_block: snapshot.inclusion_block, dump: LeasedDump { @@ -353,7 +365,14 @@ fn baseline_snapshot_in(conn: &Connection) -> Result> { .optional() } -pub(super) fn finalized_dump_in(conn: &Connection) -> Result> { +pub(super) fn finalized_dump_in( + conn: &Connection, +) -> std::result::Result, FinalizedSelectionError> { + // A matched batch can precede a divergent accepted batch in the same L1 + // block. Its snapshot cannot represent that complete block boundary. + if super::safe_accepted_batches::canonical_divergence_in(conn)?.is_some() { + return Err(FinalizedSelectionError::CanonicalDivergence); + } if let Some((batch_index, nonce, inclusion_block)) = latest_accepted_boundary_in(conn)? { let snapshot = snapshot_for_batch_in(conn, batch_index)?; return Ok(Some(FinalizedDump { @@ -548,7 +567,9 @@ mod tests { .unwrap(); assert!(matches!( storage.finalized_dump(), - Err(rusqlite::Error::QueryReturnedNoRows) + Err(FinalizedSelectionError::Storage( + rusqlite::Error::QueryReturnedNoRows + )) )); assert!(matches!( storage.latest_snapshot(), From 5ea6afd4750d86f5357a84cf40366fd8f311a7e1 Mon Sep 17 00:00:00 2001 From: gcdepaula Date: Fri, 18 Sep 2026 20:15:56 -0300 Subject: [PATCH 19/29] fix: classify snapshot artifact errors through streamed reads --- sequencer/src/egress/api/snapshot.rs | 118 ++++++++++++++++-- .../src/ingress/inclusion_lane/dump_info.rs | 10 +- 2 files changed, 115 insertions(+), 13 deletions(-) diff --git a/sequencer/src/egress/api/snapshot.rs b/sequencer/src/egress/api/snapshot.rs index f5e98c8f..0d802b12 100644 --- a/sequencer/src/egress/api/snapshot.rs +++ b/sequencer/src/egress/api/snapshot.rs @@ -121,7 +121,7 @@ async fn finalized_state( let history = leased.history_version; let LeasedDump { guard, .. } = leased; - match File::open(&path).await { + match comparison_io(File::open(&path).await, &path) { Ok(file) => Response::builder() .status(StatusCode::OK) .header(header::CONTENT_TYPE, "application/octet-stream") @@ -133,17 +133,10 @@ async fn finalized_state( "X-Recovery-Generation", history.recovery_generation.get().to_string(), ) - .body(stream_body(file, guard)) + .body(stream_body(file, guard, path)) .expect("snapshot response headers are well-formed"), // `guard` is a local here; on this error path it drops → lease released. - Err(err) => { - if err.kind() == std::io::ErrorKind::NotFound { - abort_terminal(format!( - "durable finalized snapshot artifact missing: {path:?}" - )); - } - internal_error("open finalized state file", err) - } + Err(err) => internal_error("open finalized state file", err), } } @@ -249,13 +242,43 @@ fn state_file_path(state: &SnapshotState, prefix: &Path) -> PathBuf { .unwrap_or_else(|_| abort_terminal("application snapshot path callback panicked")) } -fn stream_body(file: File, guard: LeaseGuard) -> Body { +fn stream_body(file: File, guard: LeaseGuard, path: PathBuf) -> Body { Body::from_stream(ReaderStream::new(GuardedReader { - file, + file: ComparisonReader { reader: file, path }, _guard: Arc::new(guard), })) } +fn comparison_io(result: std::io::Result, path: &Path) -> std::io::Result { + if let Err(error) = &result + && dump_info::referenced_artifact_io_is_terminal(error) + { + abort_terminal(format!( + "durable comparison artifact is unusable: {}: {error}", + path.display() + )); + } + result +} + +struct ComparisonReader { + reader: R, + path: PathBuf, +} + +impl AsyncRead for ComparisonReader { + fn poll_read( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &mut ReadBuf<'_>, + ) -> Poll> { + let this = self.get_mut(); + Pin::new(&mut this.reader) + .poll_read(cx, buf) + .map(|result| comparison_io(result, &this.path)) + } +} + /// Do not turn a producer failure into a successful truncated archive response. struct ArchiveReader { reader: tokio::io::DuplexStream, @@ -388,6 +411,77 @@ mod tests { use super::*; use crate::storage::test_helpers::temp_db; + async fn read_corrupt_comparison(path: &Path) { + let db = temp_db("corrupt-comparison-path"); + let mut storage = Storage::open(&db.path).unwrap(); + storage + .insert_baseline_snapshot(path, crate::storage::ExecutedInputCount::ZERO) + .unwrap(); + let state = Arc::new(SnapshotApiState { + snapshot: SnapshotState { + db_path: db.path, + state_file_in_dump: Path::to_path_buf, + }, + shutdown: RuntimeScope::default(), + release_scheduler: Arc::new(|release| release()), + }); + let response = finalized_state(State(state), HeaderMap::new()).await; + // Unix can open a directory successfully: the structural error first + // appears when the response body reads it. + let _ = axum::body::to_bytes(response.into_body(), 1024).await; + panic!("corrupt comparison artifact did not abort"); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + #[cfg(unix)] + async fn comparison_directory_aborts_when_streamed() { + if !crate::runtime::shutdown::abort_test_child( + "egress::api::snapshot::tests::comparison_directory_aborts_when_streamed", + ) { + return; + } + let root = tempfile::tempdir().unwrap(); + read_corrupt_comparison(root.path()).await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + #[cfg(unix)] + async fn comparison_file_parent_aborts_on_open() { + if !crate::runtime::shutdown::abort_test_child( + "egress::api::snapshot::tests::comparison_file_parent_aborts_on_open", + ) { + return; + } + let root = tempfile::tempdir().unwrap(); + let parent = root.path().join("file"); + std::fs::write(&parent, b"not a directory").unwrap(); + read_corrupt_comparison(&parent.join("comparison")).await; + } + + #[tokio::test] + async fn comparison_operational_read_error_remains_nonterminal() { + struct Unavailable; + impl AsyncRead for Unavailable { + fn poll_read( + self: Pin<&mut Self>, + _: &mut Context<'_>, + _: &mut ReadBuf<'_>, + ) -> Poll> { + Poll::Ready(Err(std::io::Error::from( + std::io::ErrorKind::PermissionDenied, + ))) + } + } + let mut reader = ComparisonReader { + reader: Unavailable, + path: PathBuf::from("temporarily-unavailable"), + }; + let error = tokio::io::AsyncReadExt::read(&mut reader, &mut [0; 1]) + .await + .unwrap_err(); + assert_eq!(error.kind(), std::io::ErrorKind::PermissionDenied); + } + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[cfg(unix)] async fn finalized_state_path_panic_aborts_process() { diff --git a/sequencer/src/ingress/inclusion_lane/dump_info.rs b/sequencer/src/ingress/inclusion_lane/dump_info.rs index 8bcc7c6c..98cf12dd 100644 --- a/sequencer/src/ingress/inclusion_lane/dump_info.rs +++ b/sequencer/src/ingress/inclusion_lane/dump_info.rs @@ -100,7 +100,7 @@ pub(crate) fn write_archive( let mut archive = tar::Builder::new(writer); archive.append_path_with_name(dump_dir.join(INFO_FILE), INFO_FILE)?; let state = app_prefix(dump_dir); - if state.is_dir() { + if std::fs::metadata(&state)?.is_dir() { archive.append_dir_all(APP_STATE_SUBDIR, state)?; } else { archive.append_path_with_name(state, APP_STATE_SUBDIR)?; @@ -441,6 +441,14 @@ mod tests { check::(); } + #[test] + fn archive_propagates_missing_state_metadata() { + let root = tempfile::tempdir().unwrap(); + write_info(root.path(), &DumpInfo::at_baseline(0)).unwrap(); + let error = write_archive(Vec::new(), root.path(), 0, None).unwrap_err(); + assert_eq!(error.kind(), io::ErrorKind::NotFound); + } + fn sample() -> DumpInfo { DumpInfo { format_version: FORMAT_VERSION, From 60966aebe948498fdb61aa054ae3cd420f820869 Mon Sep 17 00:00:00 2001 From: gcdepaula Date: Fri, 18 Sep 2026 20:18:34 -0300 Subject: [PATCH 20/29] fix: require recovery stop to cover the trusted checkpoint --- docs/recovery/cockroach.md | 18 +- sequencer/src/commands/error.rs | 21 +- .../src/commands/setup/checkpoint_tests.rs | 226 ++++++++++++++++++ sequencer/src/commands/setup/mod.rs | 32 ++- sequencer/src/recovery/mod.rs | 8 + 5 files changed, 293 insertions(+), 12 deletions(-) create mode 100644 sequencer/src/commands/setup/checkpoint_tests.rs diff --git a/docs/recovery/cockroach.md b/docs/recovery/cockroach.md index 38d07d15..86aff380 100644 --- a/docs/recovery/cockroach.md +++ b/docs/recovery/cockroach.md @@ -9,7 +9,7 @@ rebuilding.** The operator initiates recovery; the command automates the rebuild The procedure is **flush → fold → fill**: 1. **Flush** outstanding submitter transactions and choose a fixed safe L1 - stopping block. + stopping block at or after the trusted checkpoint's inclusion block. 2. **Fold** the input history through the canonical scheduler, starting from the trusted checkpoint. Every input receives its normal scheduler treatment: accepted batches execute, malformed or rejected batches are skipped, and @@ -169,7 +169,9 @@ cargo run -p wallet-sequencer -- setup --recovery \ Recovery signs L1 transactions, so the key must match the configured submitter. After success, start `run` with that same data directory. A completed rebuild refuses another `setup --recovery`; failures before completion publish no partial -baseline. +baseline. If the RPC node has not reached the checkpoint, recovery exits with +retryable code 20. Synchronize that node and retry with the same checkpoint and +incomplete data directory. ## Implementation contract @@ -196,6 +198,11 @@ accepted batch in block `B` could still be pending but disappear from the seed range. Checkpoint state and nonce remain operator-trusted; the later content-identity check does not verify this prefix. +The complete ordering is `A < B <= C`, with `A = B = 0` allowed for empty +genesis. Before sourcing or executing the fold, recovery requires `B <= C`. +Otherwise publishing the checkpoint state at an earlier baseline block could +make normal reconciliation execute already-accounted direct inputs again. + ### Flush and stopping block The lost database cannot supply its previous wallet-nonce watermark. Flushing @@ -208,6 +215,13 @@ After flushing, raw L1 ingestion must reach at least `C`. It may advance farther but the fold stops at `C`. Accepted-batch projection is deferred until the new baseline and batch tree exist. +A trusted checkpoint can be ahead of an honest replacement node that is still +synchronizing. A successful flush only settles the wallet slots known to that +node; it does not establish `C >= B`. If `C < B`, recovery refuses with retryable +exit 20, even when the later re-sync head has reached `B`: that newer observation +does not replace the fixed stopping block. It publishes no baseline and must be +retried after the node catches up. + ### Replay boundaries Seed the scheduler's pending-direct queue from `(A, B]`, excluding inputs sent by diff --git a/sequencer/src/commands/error.rs b/sequencer/src/commands/error.rs index 37aaf820..5ed45a76 100644 --- a/sequencer/src/commands/error.rs +++ b/sequencer/src/commands/error.rs @@ -386,8 +386,9 @@ pub enum BootstrapError { } /// Terminal failures of the `setup --recovery` procedure — the ones -/// an operator must resolve (the flush and the post-flush re-sync reuse the -/// transient [`RecoveryError`] paths instead). All map to [`EXIT_TERMINAL`]: +/// an operator must resolve (flush, post-flush re-sync, and a stopping block +/// behind the checkpoint use the transient [`RecoveryError`] paths instead). +/// All map to [`EXIT_TERMINAL`]: /// a plain restart re-runs the same bad inputs and re-fails identically. #[derive(Debug, Error)] pub enum SetupRecoveryError { @@ -432,11 +433,10 @@ pub enum SetupRecoveryError { /// `setup`'s read-only detection gate: the reasons a /// fresh `setup` refuses because a *previous* instance left work past the -/// checkpoint. Because plain setup has already initialized a genesis baseline, -/// the remedy is to wipe that uncompleted data dir and run `setup --recovery` -/// which flushes/folds the outstanding batches; a plain `setup` restart -/// re-detects and re-refuses (hence [`EXIT_SETUP_NEEDS_RECOVERY`], not the -/// auto-recovery class 10). +/// checkpoint. The gate runs before baseline publication and leaves setup +/// incomplete. Rebuild in a fresh data directory with `setup --recovery` to +/// flush/fold the outstanding batches; a plain `setup` restart re-detects and +/// re-refuses (hence [`EXIT_SETUP_NEEDS_RECOVERY`], not auto-recovery class 10). /// /// Both variants carry diagnostic fields for the refusal log line. #[derive(Debug, Error)] @@ -839,6 +839,13 @@ mod tests { }), "a resync behind the flush's observed view", ), + ( + recovery_retry(RecoveryRetryReason::CheckpointAheadOfStop { + checkpoint_block: 901, + stop_block: 899, + }), + "a valid manual recovery checkpoint ahead of the RPC stopping block", + ), ( CommandError::Bootstrap(BootstrapError::Identity( IdentityError::FirstBootRequiresL1, diff --git a/sequencer/src/commands/setup/checkpoint_tests.rs b/sequencer/src/commands/setup/checkpoint_tests.rs new file mode 100644 index 00000000..14d3ba87 --- /dev/null +++ b/sequencer/src/commands/setup/checkpoint_tests.rs @@ -0,0 +1,226 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +use std::path::{Path, PathBuf}; + +use alloy_primitives::{Address, U256}; +use app_core::application::{WalletApp, WalletConfig}; +use sequencer_core::application::{ + AppError, AppOutputs, Application, ApplicationProgress, ValidationOutcome, execute_direct_input, +}; +use sequencer_core::history::ExecutedInputCount; +use sequencer_core::l2_tx::{DirectInput, ValidUserOp}; +use sequencer_core::user_op::UserOp; + +use super::{Checkpoint, rebuild_from_checkpoint}; +use crate::commands::error::{BootstrapError, CommandError, EXIT_RESTART_TRANSIENT}; +use crate::ingress::inclusion_lane::dump_info; +use crate::recovery::{RecoveryError, RecoveryFailure, RecoveryRetryReason}; +use crate::storage::test_helpers::{ + SENDER_A, default_protocol_timing, pin_test_deployment_identity, temp_db, +}; +use crate::storage::{FrontierMode, LifecycleCommand, Storage, StoredSafeInput}; + +struct ReplayForbiddenApp(ApplicationProgress); + +impl Application for ReplayForbiddenApp { + fn max_method_payload_bytes() -> usize { + 0 + } + + fn validate_user_op( + &self, + _: Address, + _: &UserOp, + _: u16, + ) -> Result { + panic!("an uncovered checkpoint must refuse before application execution") + } + + fn apply_valid_user_op(&mut self, _: &ValidUserOp, _: u64) -> Result { + panic!("an uncovered checkpoint must refuse before application execution") + } + + fn apply_direct_input(&mut self, _: &DirectInput) -> Result { + panic!("an uncovered checkpoint must refuse before application execution") + } + + fn progress(&self) -> ApplicationProgress { + self.0 + } + + fn from_dump(_: &Path) -> Result { + unreachable!("the checkpoint is supplied by the fixture") + } + + fn create_dump(&mut self, _: &Path) -> Result<(), AppError> { + panic!("an uncovered checkpoint must refuse before artifact creation") + } + + fn state_file_in_dump(_: &Path) -> PathBuf { + unreachable!("the refused checkpoint has no new artifact") + } +} + +#[test] +fn recovery_refuses_stop_before_checkpoint_without_replay_or_publication() { + // The last case has H1 >= B: resync reaching the checkpoint cannot replace + // the fixed fold boundary C. The earlier cases model an honestly lagging node. + for (application_clock, checkpoint_block, stop_block, resynced_head) in [ + (900, 901, 899, 899), + (900, 964, 930, 930), + (900, 964, 930, 964), + ] { + let db = temp_db("recovery-uncovered-checkpoint"); + let mut storage = + Storage::initialize_for_command(&db.path, LifecycleCommand::Rebuild).unwrap(); + pin_test_deployment_identity(&mut storage, SENDER_A); + let identity = storage.deployment_identity().unwrap().unwrap(); + let inputs = if resynced_head > application_clock { + vec![StoredSafeInput { + sender: Address::repeat_byte(0x22), + payload: vec![1], + block_number: application_clock + 1, + }] + } else { + vec![] + }; + storage + .append_safe_inputs_with_timestamp( + resynced_head, + resynced_head, + &inputs, + SENDER_A, + &default_protocol_timing(), + FrontierMode::DeferUntilAnchorSet, + ) + .unwrap(); + let checkpoint = Checkpoint { + app: ReplayForbiddenApp( + ApplicationProgress::try_new(ExecutedInputCount::new(1), application_clock) + .unwrap(), + ), + executed_safe_block: application_clock, + checkpoint_nonce: 1, + checkpoint_block, + }; + let dumps = tempfile::tempdir().unwrap(); + + let error = rebuild_from_checkpoint( + checkpoint, + &identity, + stop_block, + &mut storage, + dumps.path(), + ) + .expect_err("the RPC stopping block must cover the trusted checkpoint"); + + assert_eq!(error.exit_code(), EXIT_RESTART_TRANSIENT); + assert!(matches!( + error, + CommandError::Bootstrap(BootstrapError::Recovery(RecoveryError::Retry(ref failure))) + if matches!(failure.as_ref(), RecoveryFailure::PolicyRetry( + RecoveryRetryReason::CheckpointAheadOfStop { + checkpoint_block: found_checkpoint, + stop_block: found_stop, + }) if *found_checkpoint == checkpoint_block && *found_stop == stop_block) + )); + assert!(!storage.is_setup_complete().unwrap()); + assert!(matches!( + storage.history_state(), + Err(rusqlite::Error::QueryReturnedNoRows) + )); + assert!(storage.open_state().unwrap().is_none()); + assert!(storage.latest_snapshot().unwrap().is_none()); + assert_eq!(std::fs::read_dir(dumps.path()).unwrap().count(), 0); + assert_eq!(storage.current_safe_block().unwrap(), Some(resynced_head)); + assert_eq!( + storage.safe_input_end_exclusive().unwrap(), + inputs.len() as u64 + ); + } +} + +#[test] +fn recovery_publishes_at_checkpoint_or_later_including_genesis() { + let owner = Address::repeat_byte(0x77); + for (checkpoint_block, stop_block, expected_count, expected_balance) in + [(10, 10, 2, 120_u64), (10, 15, 3, 150), (0, 0, 0, 0)] + { + let db = temp_db("recovery-covered-checkpoint"); + let mut storage = + Storage::initialize_for_command(&db.path, LifecycleCommand::Rebuild).unwrap(); + pin_test_deployment_identity(&mut storage, SENDER_A); + let identity = storage.deployment_identity().unwrap().unwrap(); + let config = WalletConfig::devnet(); + let deposit = |block, amount: u64| DirectInput { + sender: config.erc20_portal_address, + block_number: block, + payload: [ + config.supported_erc20_token.as_slice(), + owner.as_slice(), + U256::from(amount).to_be_bytes::<32>().as_slice(), + ] + .concat(), + }; + let mut app = WalletApp::new(config.clone()); + if checkpoint_block != 0 { + execute_direct_input(&mut app, &deposit(5, 100)).unwrap(); + } + let checkpoint = Checkpoint { + executed_safe_block: app.last_executed_safe_block(), + app, + checkpoint_nonce: u64::from(checkpoint_block != 0), + checkpoint_block, + }; + let resynced_head = stop_block + 5; + let inputs = [(5, 100), (7, 20), (12, 30), (16, 40)] + .into_iter() + .filter(|(block, _)| *block <= resynced_head) + .map(|(block, amount)| { + let input = deposit(block, amount); + StoredSafeInput { + sender: input.sender, + payload: input.payload, + block_number: block, + } + }) + .collect::>(); + storage + .append_safe_inputs_with_timestamp( + resynced_head, + resynced_head, + &inputs, + SENDER_A, + &default_protocol_timing(), + FrontierMode::DeferUntilAnchorSet, + ) + .unwrap(); + let dumps = tempfile::tempdir().unwrap(); + + rebuild_from_checkpoint( + checkpoint, + &identity, + stop_block, + &mut storage, + dumps.path(), + ) + .unwrap(); + + assert!(storage.is_setup_complete().unwrap()); + let history = storage.history_state().unwrap(); + assert_eq!(history.base_safe_block, stop_block); + assert_eq!(history.base_executed_input_count, expected_count); + assert_eq!( + storage.open_state().unwrap().unwrap().safe_block, + stop_block + ); + let snapshot = storage.latest_snapshot().unwrap().unwrap(); + let app = WalletApp::from_dump(&dump_info::app_prefix(&snapshot.dump.prefix)).unwrap(); + assert_eq!(app.executed_input_count().get(), expected_count); + assert_eq!( + app.current_user_balance(owner), + U256::from(expected_balance) + ); + } +} diff --git a/sequencer/src/commands/setup/mod.rs b/sequencer/src/commands/setup/mod.rs index a6725e70..df2dea7c 100644 --- a/sequencer/src/commands/setup/mod.rs +++ b/sequencer/src/commands/setup/mod.rs @@ -28,6 +28,8 @@ use alloy_primitives::Address; use sequencer_core::application::{AppError, Application}; use sequencer_core::scheduler::{FoldInput, SchedulerConfig, fold_replay}; +#[cfg(test)] +mod checkpoint_tests; pub(crate) mod fill; use super::{ensure_deployment_identity, validate_rpc_chain_id}; @@ -37,7 +39,9 @@ use crate::commands::error::{ }; use crate::ingress::inclusion_lane::dump_info; use crate::l1::reader::{InputReader, InputReaderConfig, InputReaderError}; -use crate::recovery::{MempoolFlusher, assert_resync_caught_up}; +use crate::recovery::{ + MempoolFlusher, RecoveryError, RecoveryRetryReason, assert_resync_caught_up, +}; use crate::storage::{self, DeploymentIdentity, FeeOracleIdentity}; pub async fn setup(config: SetupConfig, genesis_app: F) -> Result<(), CommandError> @@ -495,8 +499,8 @@ fn source_fold_inputs( /// The `setup --recovery` procedure: rebuild a freshly-wiped DB from /// a trusted checkpoint instead of refusing. Runs after the shared prefix /// (identity pinned, initial sync done); replaces the detection gate + genesis -/// snapshot. Distinct, terminal error type ([`SetupRecoveryError`]) from -/// `run`'s recovery — operator-driven, one-shot. +/// snapshot. Invalid checkpoint/configuration failures are terminal; transient +/// L1 failures leave setup incomplete for a fresh attempt. /// /// The `flush → fold → fill` steps are enumerated authoritatively in /// **[`docs/recovery/cockroach.md`](../../../docs/recovery/cockroach.md)** (spec, @@ -548,6 +552,28 @@ where let resynced_safe_block = require_resynced_safe_block(storage.current_safe_block()?)?; assert_resync_caught_up(resynced_safe_block, stop_block)?; + rebuild_from_checkpoint(checkpoint, identity, stop_block, storage, dumps_dir) +} + +fn rebuild_from_checkpoint( + checkpoint: Checkpoint, + identity: &DeploymentIdentity, + stop_block: u64, + storage: &mut storage::Storage, + dumps_dir: &std::path::Path, +) -> Result<(), CommandError> { + // A valid checkpoint can outpace an honestly lagging RPC node. Labelling + // its state with an earlier C would let run execute its directs again. + if stop_block < checkpoint.checkpoint_block { + return Err( + RecoveryError::retry(RecoveryRetryReason::CheckpointAheadOfStop { + checkpoint_block: checkpoint.checkpoint_block, + stop_block, + }) + .into(), + ); + } + // 4. Source the (A, B] direct seeds + the (B, C] replay stream. let submitter = identity.batch_submitter_address; let (seeds, replay) = source_fold_inputs(storage, &checkpoint, stop_block, submitter)?; diff --git a/sequencer/src/recovery/mod.rs b/sequencer/src/recovery/mod.rs index f862ac79..48bb046f 100644 --- a/sequencer/src/recovery/mod.rs +++ b/sequencer/src/recovery/mod.rs @@ -76,6 +76,14 @@ pub enum RecoveryRetryReason { resynced_safe_block: u64, flush_observed_safe_block: u64, }, + #[error( + "post-flush stopping block {stop_block} predates checkpoint block {checkpoint_block}; \ + synchronize the RPC node and retry recovery" + )] + CheckpointAheadOfStop { + checkpoint_block: u64, + stop_block: u64, + }, #[error("local recovery facts changed before phase execution: {status:?}")] StaleDecision { status: DangerStatus }, #[error("the Tip was already open when the EnsureOpenTip phase ran")] From e45fc96f0ef89c889e497d947f43d31d4d8b19f8 Mon Sep 17 00:00:00 2001 From: gcdepaula Date: Fri, 18 Sep 2026 20:23:25 -0300 Subject: [PATCH 21/29] fix: bound snapshot response headers by the SDK timeout --- sdk/rust-client/src/errors.rs | 2 + sdk/rust-client/src/lib.rs | 71 ++++++++++++++++++++++++++++++++--- 2 files changed, 67 insertions(+), 6 deletions(-) diff --git a/sdk/rust-client/src/errors.rs b/sdk/rust-client/src/errors.rs index cd7ed739..cabe5673 100644 --- a/sdk/rust-client/src/errors.rs +++ b/sdk/rust-client/src/errors.rs @@ -79,6 +79,8 @@ pub enum SubscribeError { #[derive(Debug, Error)] pub enum SnapshotError { + #[error("snapshot response headers timed out")] + HeadersTimeout, #[error("snapshot request failed: {0}")] Request(#[from] reqwest::Error), #[error("invalid snapshot metadata: {0}")] diff --git a/sdk/rust-client/src/lib.rs b/sdk/rust-client/src/lib.rs index 79302ecd..fefee4e7 100644 --- a/sdk/rust-client/src/lib.rs +++ b/sdk/rust-client/src/lib.rs @@ -143,16 +143,18 @@ impl SequencerClient { serde_json::from_str::(&body).map_err(|e| GetFeeError::Decode(e.to_string())) } - /// Streams without the short transaction deadline; callers own download cancellation. + /// Bounds response headers by the request timeout; callers own body cancellation. pub async fn latest_snapshot(&self) -> Result { - let response = self + let request = self .http_client .get(format!( "{}/latest_snapshot", self.endpoint.trim_end_matches('/') )) - .send() - .await? + .send(); + let response = tokio::time::timeout(self.request_timeout, request) + .await + .map_err(|_| SnapshotError::HeadersTimeout)?? .error_for_status()?; let header = |name| { response @@ -302,6 +304,28 @@ mod tests { )); } + #[tokio::test] + async fn snapshot_headers_keep_the_configured_request_timeout() { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let client = SequencerClient::new(format!("http://{address}")) + .unwrap() + .with_request_timeout(Duration::from_millis(100)); + let stalled_server = async { + let (_stream, _) = listener.accept().await.unwrap(); + std::future::pending::<()>().await; + }; + let result = tokio::time::timeout(Duration::from_secs(2), async { + tokio::select! { + result = client.latest_snapshot() => result, + () = stalled_server => unreachable!(), + } + }) + .await + .expect("an accepted connection with no headers must reach the configured deadline"); + assert!(matches!(result, Err(SnapshotError::HeadersTimeout))); + } + #[tokio::test] async fn snapshot_body_outlives_the_transaction_request_timeout() { use tokio::io::{AsyncReadExt, AsyncWriteExt}; @@ -317,12 +341,12 @@ mod tests { received += count; } stream.write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 4\r\nX-History-Era: 00112233-4455-4677-8899-aabbccddeeff\r\nX-Recovery-Generation: 0\r\nX-Executed-Input-Count: 7\r\n\r\n").await.unwrap(); - tokio::time::sleep(Duration::from_millis(100)).await; + tokio::time::sleep(Duration::from_millis(300)).await; stream.write_all(b"dump").await.unwrap(); }); let client = SequencerClient::new_with_timeout( format!("http://{address}"), - Duration::from_millis(30), + Duration::from_millis(100), ) .unwrap(); let snapshot = client.latest_snapshot().await.unwrap(); @@ -364,4 +388,39 @@ mod tests { SubscribeError::History(actual) if actual == policy) ); } + + #[tokio::test] + async fn subscription_below_nonzero_baseline_decodes_history_unavailable() { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { + let (mut stream, _) = listener.accept().await.unwrap(); + let mut request = Vec::new(); + while !request.ends_with(b"\r\n\r\n") { + let mut byte = [0]; + assert_eq!(stream.read(&mut byte).await.unwrap(), 1); + request.push(byte[0]); + assert!(request.len() <= 8192); + } + let request = String::from_utf8(request).unwrap(); + assert!(request.starts_with("GET /ws/subscribe?")); + assert!(request.contains("next_input=40 ")); + stream.write_all(b"HTTP/1.1 409 Conflict\r\nX-History-Error: {\"code\":\"HISTORY_UNAVAILABLE\",\"available_from\":41}\r\nContent-Length: 0\r\nConnection: close\r\n\r\n").await.unwrap(); + }); + let client = SequencerClient::new(format!("http://{address}")).unwrap(); + let result = client + .subscribe(HistoryClaim { + version: HistoryVersion { + era_id: "00112233-4455-4677-8899-aabbccddeeff".parse().unwrap(), + recovery_generation: RecoveryGeneration::new(0), + }, + next_input: ExecutedInputCount::new(40), + }) + .await; + assert!(matches!(result, + Err(SubscribeError::History(HistoryPolicyError::HistoryUnavailable { available_from })) + if available_from == ExecutedInputCount::new(41))); + server.await.unwrap(); + } } From 18e99e4fe2d03426954f72df5641196d12103e99 Mon Sep 17 00:00:00 2001 From: gcdepaula Date: Fri, 18 Sep 2026 20:23:25 -0300 Subject: [PATCH 22/29] test: pin snapshot failure and history recovery boundaries --- sequencer-core/src/history.rs | 42 ++++++++++++++++ sequencer/src/commands/run/startup_hygiene.rs | 46 +++++++++++++++++ .../src/commands/setup/checkpoint_tests.rs | 2 +- sequencer/src/ingress/inclusion_lane/tests.rs | 42 ++++++++++++++++ .../integration_tests/historical_bootstrap.rs | 8 +++ .../src/storage/egress/historical/tests.rs | 6 +-- sequencer/src/storage/history.rs | 49 ++++++++++--------- sequencer/src/storage/snapshot_dumps.rs | 15 ++++++ tests/e2e/src/cold_replica.rs | 23 ++++++++- 9 files changed, 204 insertions(+), 29 deletions(-) diff --git a/sequencer-core/src/history.rs b/sequencer-core/src/history.rs index dbd1b7fa..abdf938e 100644 --- a/sequencer-core/src/history.rs +++ b/sequencer-core/src/history.rs @@ -258,6 +258,48 @@ mod tests { assert!("00112233445546778899aabbccddeeff".parse::().is_err()); } + #[test] + fn history_policy_errors_preserve_literal_wire_codes_and_fields() { + let current = HistoryVersion { + era_id: CANONICAL.parse().unwrap(), + recovery_generation: RecoveryGeneration::new(7), + }; + for (error, json) in [ + ( + HistoryPolicyError::EraChanged { current }, + serde_json::json!({ + "code": "ERA_CHANGED", + "current": { "era_id": CANONICAL, "recovery_generation": 7 } + }), + ), + ( + HistoryPolicyError::StaleGeneration { current }, + serde_json::json!({ + "code": "STALE_GENERATION", + "current": { "era_id": CANONICAL, "recovery_generation": 7 } + }), + ), + ( + HistoryPolicyError::HistoryUnavailable { + available_from: ExecutedInputCount::new(41), + }, + serde_json::json!({ "code": "HISTORY_UNAVAILABLE", "available_from": 41 }), + ), + ( + HistoryPolicyError::AheadOfHead { + head: ExecutedInputCount::new(50), + }, + serde_json::json!({ "code": "AHEAD_OF_HEAD", "head": 50 }), + ), + ] { + assert_eq!(serde_json::to_value(error).unwrap(), json); + assert_eq!( + serde_json::from_value::(json).unwrap(), + error + ); + } + } + #[test] fn era_id_displays_canonical_lowercase_hyphenated_form() { let era = EraId::from_bytes(CANONICAL_BYTES).expect("canonical UUIDv4"); diff --git a/sequencer/src/commands/run/startup_hygiene.rs b/sequencer/src/commands/run/startup_hygiene.rs index e6005f4b..0271aeb4 100644 --- a/sequencer/src/commands/run/startup_hygiene.rs +++ b/sequencer/src/commands/run/startup_hygiene.rs @@ -183,6 +183,52 @@ mod tests { assert_eq!(removed, 0); } + #[test] + fn startup_resets_persisted_crash_leases_before_collecting_artifacts() { + let db = temp_db("startup-crash-leases"); + let mut storage = Storage::open(&db.path).unwrap(); + let dumps = tempfile::tempdir().unwrap(); + let baseline = dumps.path().join("baseline"); + create_structured_dump(&baseline); + let baseline_id = storage + .insert_baseline_snapshot(&baseline, crate::storage::ExecutedInputCount::ZERO) + .unwrap(); + let obsolete = dumps.path().join("obsolete"); + create_structured_dump(&obsolete); + let obsolete_id = storage + .write(|tx| { + tx.execute( + "UPDATE dumps SET lease_count = 1 WHERE id = ?1", + [baseline_id], + )?; + tx.execute( + "INSERT INTO dumps(prefix, lease_count) VALUES (?1, 1)", + [obsolete.to_str().unwrap()], + )?; + Ok(tx.last_insert_rowid()) + }) + .unwrap(); + drop(storage); + + let mut storage = Storage::open(&db.path).unwrap(); + assert_eq!(storage.dump_lease_count(obsolete_id).unwrap(), Some(1)); + assert!(storage.gc_unreferenced_dumps().unwrap().is_empty()); + assert!(obsolete.exists(), "the persisted lease blocks ordinary GC"); + + run_snapshot_hygiene(&mut storage, dumps.path()).unwrap(); + + assert_eq!(storage.dump_lease_count(baseline_id).unwrap(), Some(0)); + assert_eq!(storage.dump_lease_count(obsolete_id).unwrap(), None); + assert!( + baseline.exists(), + "the rollback baseline survives startup GC" + ); + assert!( + !obsolete.exists(), + "startup collects the abandoned leased artifact" + ); + } + #[test] fn snapshot_gc_at_startup_removes_unreferenced_rows() { let db = temp_db("gc-startup"); diff --git a/sequencer/src/commands/setup/checkpoint_tests.rs b/sequencer/src/commands/setup/checkpoint_tests.rs index 14d3ba87..21f603e4 100644 --- a/sequencer/src/commands/setup/checkpoint_tests.rs +++ b/sequencer/src/commands/setup/checkpoint_tests.rs @@ -163,7 +163,7 @@ fn recovery_publishes_at_checkpoint_or_later_including_genesis() { ] .concat(), }; - let mut app = WalletApp::new(config.clone()); + let mut app = WalletApp::new(config); if checkpoint_block != 0 { execute_direct_input(&mut app, &deposit(5, 100)).unwrap(); } diff --git a/sequencer/src/ingress/inclusion_lane/tests.rs b/sequencer/src/ingress/inclusion_lane/tests.rs index b62ef352..9f58bfa0 100644 --- a/sequencer/src/ingress/inclusion_lane/tests.rs +++ b/sequencer/src/ingress/inclusion_lane/tests.rs @@ -56,6 +56,7 @@ fn decode_progress(bytes: &[u8], app_name: &str) -> Result, progress: ApplicationProgress, + fail_dump: bool, /// Test-only scheduling seam used to keep a rejected queue saturated long /// enough to distinguish one bounded turn from an unbounded drain. reject_user_ops_after: Option, @@ -113,6 +114,9 @@ impl Application for TestApp { } fn create_dump(&mut self, prefix: &Path) -> Result<(), AppError> { + if self.fail_dump { + return Err(std::io::Error::other("injected application dump failure").into()); + } std::fs::create_dir(prefix)?; std::fs::write(Self::state_file_in_dump(prefix), b"")?; Ok(()) @@ -1955,6 +1959,44 @@ async fn restart_resumes_from_pending_checkpoint_without_skipping_txs() { ); } +#[test] +fn application_dump_failure_leaves_tip_and_latest_snapshot_unchanged() { + let db = temp_db("application-dump-failure"); + let mut storage = Storage::open(&db.path).unwrap(); + let mut head = storage + .initialize_open_state(0, SafeInputRange::empty_at(0)) + .unwrap(); + let dumps = tempfile::tempdir().unwrap(); + let mut app = TestApp::default(); + register_genesis_snapshot(&mut app, &mut storage, dumps.path()); + super::snapshot::close_batch_with_snapshot(&mut app, &mut storage, &mut head, 0, dumps.path()) + .unwrap(); + let snapshot = storage.latest_snapshot().unwrap().unwrap(); + let tip = head.batch_index; + let dump_count = storage.list_dump_rows().unwrap().len(); + + app.fail_dump = true; + let error = super::snapshot::close_batch_with_snapshot( + &mut app, + &mut storage, + &mut head, + 0, + dumps.path(), + ) + .expect_err("an application dump failure must precede the batch seal"); + assert!(matches!( + error, + super::snapshot::TakeDumpError::CreateDump(super::dump_info::CreateDumpDirError::App( + AppError::Io(ref source) + )) if source.to_string() == "injected application dump failure" + )); + assert_eq!(head.batch_index, tip); + assert_eq!(storage.open_state().unwrap().unwrap().batch_index, tip); + assert_eq!(storage.latest_batch_index().unwrap(), Some(tip)); + assert_eq!(storage.latest_snapshot().unwrap().unwrap(), snapshot); + assert_eq!(storage.list_dump_rows().unwrap().len(), dump_count); +} + #[test] fn empty_batch_snapshot_preserves_application_count() { let db = temp_db("empty-snapshot-count"); diff --git a/sequencer/src/integration_tests/historical_bootstrap.rs b/sequencer/src/integration_tests/historical_bootstrap.rs index bfffbd44..4cb1bbc1 100644 --- a/sequencer/src/integration_tests/historical_bootstrap.rs +++ b/sequencer/src/integration_tests/historical_bootstrap.rs @@ -294,6 +294,14 @@ async fn historical_bootstrap_restores_transfer_history_and_hands_off_at_baselin let era = metadata.history.version.era_id; assert_eq!(metadata.history.available_from.get(), 7); assert_eq!(metadata.history.head.get(), 7); + assert!(matches!( + client.subscribe(HistoryClaim { + version: metadata.history.version, + next_input: ExecutedInputCount::new(6), + }).await, + Err(sequencer_rust_client::SubscribeError::History(HistoryPolicyError::HistoryUnavailable { available_from })) + if available_from == metadata.history.available_from + )); assert_eq!(metadata.baseline.l1_stop_block, STOP); assert_eq!(metadata.baseline.l1_end_input_index, 8); assert_eq!(metadata.baseline.next_batch_nonce, reference_nonce); diff --git a/sequencer/src/storage/egress/historical/tests.rs b/sequencer/src/storage/egress/historical/tests.rs index eef4d5f9..af0b4506 100644 --- a/sequencer/src/storage/egress/historical/tests.rs +++ b/sequencer/src/storage/egress/historical/tests.rs @@ -366,9 +366,9 @@ fn accepted_checkpoint_uses_the_exact_latest_snapshot_and_preserves_baseline() { .unwrap(); assert!(matches!( storage.history_info(None, None), - Err(HistoricalReadError::Checkpoint(FinalizedSelectionError::Storage( - rusqlite::Error::QueryReturnedNoRows - ))) + Err(HistoricalReadError::Checkpoint( + FinalizedSelectionError::Storage(rusqlite::Error::QueryReturnedNoRows) + )) )); } diff --git a/sequencer/src/storage/history.rs b/sequencer/src/storage/history.rs index 0948d1db..975e70ba 100644 --- a/sequencer/src/storage/history.rs +++ b/sequencer/src/storage/history.rs @@ -3,8 +3,6 @@ //! Immutable era baseline and the preserved prefix at each recovery generation. -#[cfg(test)] -use rusqlite::OptionalExtension; use rusqlite::{Connection, Result, Transaction, params, types::Type}; use sequencer_core::history::{EraId, ExecutedInputCount, HistoryVersion, RecoveryGeneration}; @@ -65,21 +63,6 @@ pub(super) fn initialize_history_in( base: ExecutedInputCount, base_safe_block: u64, ) -> Result<()> { - #[cfg(test)] - if let Some(existing) = query_history_state(tx).optional()? { - // Test fixtures initialize genesis when opening their schema. Production - // creates this row only with the complete durable baseline. - assert_eq!( - existing.base_executed_input_count, - base.get(), - "history base differs" - ); - assert_eq!( - existing.base_safe_block, base_safe_block, - "L1 prefix differs" - ); - return Ok(()); - } let mut bytes: [u8; EraId::BYTE_LEN] = tx.query_row("SELECT randomblob(16)", [], |row| row.get(0))?; bytes[6] = (bytes[6] & 0x0f) | 0x40; @@ -182,13 +165,33 @@ mod tests { .write(|tx| initialize_history_in(tx, ExecutedInputCount::new(41), 70)) .unwrap(); let state = storage.history_state().unwrap(); - for sql in [ - "UPDATE history_state SET era_id = era_id", - "UPDATE history_state SET base_executed_input_count = 42", - "UPDATE history_state SET base_safe_block = 71", - "DELETE FROM history_state", + let duplicate = storage + .write(|tx| initialize_history_in(tx, ExecutedInputCount::new(41), 70)) + .expect_err("even identical baseline values cannot initialize another era"); + assert_eq!( + duplicate.to_string(), + "history state is inserted once per database" + ); + for (sql, expected) in [ + ( + "UPDATE history_state SET era_id = X'00000000000040008000000000000001'", + "history baseline is immutable", + ), + ( + "UPDATE history_state SET base_executed_input_count = 42", + "history baseline is immutable", + ), + ( + "UPDATE history_state SET base_safe_block = 71", + "history baseline is immutable", + ), + ( + "DELETE FROM history_state", + "history state is write-once per database", + ), ] { - assert!(storage.conn.execute(sql, []).is_err(), "{sql}"); + let error = storage.conn.execute(sql, []).expect_err(sql); + assert_eq!(error.to_string(), expected, "{sql}"); } drop(storage); let mut reopened = Storage::open(&db.path).unwrap(); diff --git a/sequencer/src/storage/snapshot_dumps.rs b/sequencer/src/storage/snapshot_dumps.rs index 9a5f6d2c..e2503dec 100644 --- a/sequencer/src/storage/snapshot_dumps.rs +++ b/sequencer/src/storage/snapshot_dumps.rs @@ -755,6 +755,21 @@ mod tests { assert_eq!(storage.dump_lease_count(id).unwrap(), Some(0)); } + #[test] + fn lease_release_cannot_underflow() { + let db = temp_db("lease-underflow"); + let mut storage = Storage::open(&db.path).unwrap(); + let id = storage + .insert_baseline_snapshot(&prefix(0), ExecutedInputCount::ZERO) + .unwrap(); + let error = storage.release_dump_lease(id).unwrap_err(); + assert_eq!( + error.sqlite_error_code(), + Some(rusqlite::ErrorCode::ConstraintViolation) + ); + assert_eq!(storage.dump_lease_count(id).unwrap(), Some(0)); + } + #[test] fn persistent_release_failure_reaches_reporter() { let db = temp_db("persistent-lease"); diff --git a/tests/e2e/src/cold_replica.rs b/tests/e2e/src/cold_replica.rs index 97d49591..6b820472 100644 --- a/tests/e2e/src/cold_replica.rs +++ b/tests/e2e/src/cold_replica.rs @@ -12,6 +12,7 @@ use rollups_harness::replay::apply_ws_message; use rollups_harness::{ManagedSequencer, ReplayWalletApp, TestSigner, WsClient}; use sequencer_core::api::WsTxMessage; use sequencer_core::application::Application; +use sequencer_core::fee::fee_to_linear; use sequencer_rust_client::{ HistoryClaim, HistoryPolicyError, SequencerClient, SnapshotResponse, SubscribeError, }; @@ -230,12 +231,30 @@ async fn run_scenario(runtime: &mut ManagedSequencer) -> Scenari assert!(recovered.executed_input_count().get() < replica.executed_input_count().get()); let mut alice_l2 = runtime.wallet_l2(TestSigner::from_default(1)?)?; - alice_l2.set_next_nonce(recovered_reference.current_user_nonce(alice_address)); - alice_l2.transfer(bob_address, U256::from(6_000)).await?; + let expected_nonce = recovered_reference.current_user_nonce(alice_address); + let balance_before = recovered_reference.current_user_balance(alice_address); + // The fixed oracle keeps this quote valid across frame rotations. Derive the + // expected debit before receiving the event so wrong feed fees cannot agree by replay. + let quote = client.get_fee().await?; + assert_eq!(quote.fee, quote.recommended_fee); + let amount = U256::from(6_000); + let expected_balance = balance_before - amount - fee_to_linear(quote.fee); + alice_l2.set_next_nonce(expected_nonce); + alice_l2.transfer(bob_address, amount).await?; let resumed = recovered_ws.expect_user_op_from(alice_address).await?; + assert!(matches!(resumed, WsTxMessage::UserOp { fee, nonce, .. } + if fee == quote.fee && nonce == expected_nonce)); apply_ws_message(&mut recovered, resumed.clone())?; recovered_reference.apply(resumed)?; assert_same_state(&mut recovered, &recovered_reference)?; + assert_eq!( + recovered_reference.current_user_nonce(alice_address), + expected_nonce + 1 + ); + assert_eq!( + recovered_reference.current_user_balance(alice_address), + expected_balance + ); assert_eq!( recovered_reference.current_user_balance(bob_address), U256::from(6_000) From f504c2e88e2b432718493e29e3c56f74fce3ad00 Mon Sep 17 00:00:00 2001 From: gcdepaula Date: Fri, 18 Sep 2026 20:28:55 -0300 Subject: [PATCH 23/29] docs: reconcile recovery contracts and record stack validation --- docs/invariants.md | 43 +++++++---- docs/review/2026-09-16-track3-validation.md | 7 ++ .../2026-09-18-stack-review-validation.md | 76 +++++++++++++++++++ docs/review/register.md | 2 + docs/snapshots/lifecycle.md | 13 +++- docs/threat-model/README.md | 8 +- docs/watchdog/design-notes.md | 18 +++-- sequencer/src/storage/open.rs | 4 +- 8 files changed, 139 insertions(+), 32 deletions(-) create mode 100644 docs/review/2026-09-18-stack-review-validation.md diff --git a/docs/invariants.md b/docs/invariants.md index 5a9b6c6e..3600002e 100644 --- a/docs/invariants.md +++ b/docs/invariants.md @@ -81,11 +81,11 @@ by writer and are write-once (`0001_schema.sql`). | inclusion lane | `batches` (insert + `sealed_at_ms`), `frames`, `user_ops`, `application_inputs`, `dumps`/`snapshots` (batch close) | | input reader | `safe_inputs`, `l1_safe_head`, `safe_accepted_batches`, `canonical_divergence` (the divergence poison marker) | | recovery (startup) | `batches.invalidated_at_ms`, Tip reopen, current `application_inputs` suffix deletion | -| history metadata (setup/recovery) | `history_state` — complete era/application-count/L1-block baseline; generation advance and immutable preserved-prefix cut in a non-empty standard-recovery cascade | +| history metadata (setup/recovery) | `history_state` — complete era/application-count/L1-block baseline and generation; `history_generation_cuts` — immutable preserved-prefix cuts written with non-empty standard-recovery cascades | | batch submitter and mempool flusher | `wallet_nonce_watermark` — deliberately shared under one protocol: each raises it before its first broadcast (write-before-broadcast, I14) | | egress (HTTP) | `dumps.lease_count` (leases); `run`'s startup hygiene resets it to zero as the crash backstop | -| setup | `deployment_identity` (pinned once), `batch_tree_anchor` (the root nonce, frozen once setup completes), the initial `dumps` + `snapshots` rows (genesis or rebuild registration, atomic with the complete history baseline), the `setup_complete` fact (written once), `batch_policy.log_gas_price` + `log_gas_price_updated_at_ms` (first write; Fixed and Uniswap) | -| snapshot GC (the lane after reconciliation, `run`'s startup hygiene) | unreferenced `dumps` row deletion (`gc_unreferenced_dumps`) | +| setup | `deployment_identity` (pinned once), `batch_tree_anchor` (the root nonce, frozen once setup completes), the initial `dumps` + `snapshots` rows and rebuild root `batches`/`frames` (atomic with the complete history baseline), the `setup_complete` fact (written once), `batch_policy.log_gas_price` + `log_gas_price_updated_at_ms` (first write; Fixed and Uniswap) | +| snapshot GC (the lane after reconciliation, `run`'s startup hygiene) | obsolete `snapshots` and unreferenced `dumps` row deletion (`gc_unreferenced_dumps`), including a superseded baseline artifact | | command brackets (run, setup, flush) | `terminal_faults` (append-only, best-effort at settlement) | | admin | `batch_policy` alpha knobs (`log_alpha`, `log_one_plus_alpha`) | | fee oracle | `batch_policy.log_gas_price` + `log_gas_price_updated_at_ms` (Uniswap mode only; stamps on every successful refresh) | @@ -224,9 +224,11 @@ by writer and are write-once (`0001_schema.sql`). remedy is cockroach recovery. - **Completeness boundary:** the check completely enforces the accepted-batch identity predicate above; it is intentionally not a general canonical/application - divergence oracle. It trusts collapsed history below the anchor and the - checkpoint application state, shares `scheduler_accepts` (including its - documented self-trust omissions), and does not independently detect bugs in + divergence oracle. The entire L1 prefix through baseline block `C` is opaque, + including previously rejected future-nonce batches; the check trusts the + checkpoint state and continuation nonce instead of reinterpreting that prefix. + It shares `scheduler_accepts` (including its documented self-trust omissions), + and does not independently detect bugs in direct-input/user-op execution. A wrong-high cockroach checkpoint nonce is a known example that can escape it. Absence of the marker therefore does not prove global agreement. Conversely, a structurally malformed foreign landing @@ -253,9 +255,11 @@ by writer and are write-once (`0001_schema.sql`). boundary selects external directs by the setup-pinned submitter address; only those inputs and included user ops enter `application_inputs`. - **Enforced by:** classified direct reads and complete receipt validation at - append. Startup/recovery derive the initial direct rows before catch-up, - which must execute them successfully before admission. Replay and WS need - no envelope filter because every row executes. + append. Standard startup recovery attributes undrained directs to the new Tip; + lane catch-up executes them before processing queued user operations. Manual + rebuild represents the folded prefix through `C` in its baseline snapshot, + with no application-history rows for that prefix. Replay and WS need no + envelope filter because every row executes. - **Depended on by:** application replay and replicated state correctness. ### I12. Safe head advances only on real observation; `synced_at_ms` is genuine progress time @@ -341,9 +345,12 @@ by writer and are write-once (`0001_schema.sql`). conflicting batch-tree writes; the detector and next typed read stop the process. A chunk committed before either runtime observation may acknowledge and later roll back. -- **Watchdog boundary:** the freeze blocks accepted-checkpoint publication before the - offending landing becomes a comparable sequencer checkpoint. Because the - watchdog skips replay when the finalized inclusion block is unchanged, it +- **Watchdog boundary:** accepted-checkpoint selection checks for divergence + in the same transaction as selection and any download lease, refusing while + the marker is present. A matching batch + before a divergent acceptance in the same L1 block cannot represent that + block's final state. Because the watchdog skips replay when the finalized + inclusion block is unchanged, it does not subsume this wire-identity detector. Conversely, the check does not subsume the watchdog's broader independent application-state comparison. @@ -429,10 +436,14 @@ by writer and are write-once (`0001_schema.sql`). before replacement directs. The entire transition commits in the cascade transaction. Clean restart changes neither token. Every intervening cut is required to authorize reusing a checkpoint from an older generation. -- **Enforced by:** `complete_baseline_setup`, immutable history triggers, - exact-`+1` generation trigger, and `cascade_and_reopen`. -- **Depended on by:** mandatory snapshot-derived WS claims. Identity is validated - before the requested count, including for empty history. +- **Enforced by:** `complete_baseline_setup`, immutable baseline and + `history_generation_cuts` triggers, the exact-`+1` generation trigger requiring + its cut, and `cascade_and_reopen`. `preserved_input_count_in` asserts that + every intervening generation has a cut before computing compatibility. +- **Depended on by:** mandatory snapshot-derived WS claims and `/history` + checkpoint compatibility across standard recoveries. Identity is validated + before the requested count, including for empty history. Cuts remain available + for the era's lifetime; their absence must never authorize a partial minimum. - **Breaks:** a client silently resumes a replaced suffix or inaccessible prefix. - **Operational boundary:** rebuilding uses a fresh/wiped data directory. Checkpoint state, inclusion block, and next nonce are trusted operator inputs; diff --git a/docs/review/2026-09-16-track3-validation.md b/docs/review/2026-09-16-track3-validation.md index 56aeaa0e..b5491328 100644 --- a/docs/review/2026-09-16-track3-validation.md +++ b/docs/review/2026-09-16-track3-validation.md @@ -5,6 +5,13 @@ Scope: validate application-history commit a complete cold replica, and a same-host latency comparison against `91e25780854bb641c63135751f951f9f7ee1e744`. +These are the measured pre-rebase revisions. Their stack counterparts are +`799a5d3` → `3e5b971` and `91e2578` → `71b3b35`; the validation/test commit +`4fc010c` became `7f3229f`. The latter stack includes upstream changes, so these +measurements must not be attributed to its rebased trees. The latency baseline +already contains the initial versioned-history foundation; it isolates the +subsequent application-history refactor, not the cost of the complete stack. + Retained for the [Track 3 integration gates](../plans/2026-07-track3-feed-replay-design.md): this is the wallet baseline against which native-engine and deployment results can be assessed. Replace or delete it when those decisions no longer use these diff --git a/docs/review/2026-09-18-stack-review-validation.md b/docs/review/2026-09-18-stack-review-validation.md new file mode 100644 index 00000000..b4845fd3 --- /dev/null +++ b/docs/review/2026-09-18-stack-review-validation.md @@ -0,0 +1,76 @@ +# Stack review closeout validation + +Evidence for reviewing and landing the closing change on +`codex/stack-review-fixes`, above reviewed tip +`62ec150f18a27697220158f1ca4d84ec4a6fee06`. The implementation and regression tests +are at `18e99e4fe2d03426954f72df5641196d12103e99`; the closing documentation commit +adds contract corrections, this record, and a storage rustdoc correction only. +Retire this record after stack landing when no ongoing review decision uses it. + +## Environment and results + +Run on macOS arm64 through the project's parent Nix/direnv shell. Cargo and +rustc were both 1.95.0, Anvil was 1.5.1, and the locally rebuilt canonical guest +used the repository-pinned Cartesi Machine 0.20.0. Local Anvil differs from CI's +1.4.3 pin; these are local results, not a new CI run. + +| Check | Result | +|---|---| +| `cargo check --locked --workspace --all-targets` | Passed | +| `cargo fmt --all -- --check` and `git diff --check` | Passed | +| `cargo clippy --locked --workspace --all-targets --all-features -- -D warnings` | Passed | +| `cargo test --locked --workspace --exclude canonical-test -- --test-threads=1` | 754 passed; one existing ignored doc test | +| `lua watchdog/tests/run.lua` | 62 passed | +| `just canonical build-machine-image`, then `cargo run --locked -p canonical-test` | Fresh image; 10 guest scheduler tests passed | +| `bash scripts/ci-c-application-smoke.sh` | External archive, generic host, and independent downstream consumer built and ran their CLI smoke checks | + +The process binaries were rebuilt after the final SDK change. Each of these +scenario filters passed for both the Rust wallet host and the C wallet host: + +- `cold_replica_snapshot_backlog_live_recovery_test` +- `restart_and_replay_test` +- `recovery_after_stale_batches_test` +- `setup_recovery_round_trip_test` + +These eight process runs include canonical watchdog comparison after recovery +and rebuild. Their prerequisites were the checksum-verified rollups-contracts +Anvil fixture, locally built test contracts, and watchdog Lua dependencies. +The first attempted process run lacked the fixture and stopped before startup; +setup supplied it before the successful runs. + +## Discriminating regressions and independent review + +- A matching local batch followed by a foreign accepted batch in the same block + returned HTTP 200 before the finalized-selection guard. The canonical scheduler + advances beyond that local snapshot. With the guard, all three finalized routes + return 503, including a conditional state request, without acquiring a lease. +- With checkpoint clock 900, checkpoint block 901, and stop 899, the unguarded + deterministic rebuild phase reached artifact creation. The new check refuses + with exit 20 before replay or publication. A later resync reaching the checkpoint + does not substitute for the fixed stop; equality and genesis still succeed. +- Without the SDK header timeout, the stalled-header regression exceeded its + outer two-second limit. With the configured header deadline restored, all + 13 SDK tests passed, including a body that outlives that deadline. + +Separate reviewers checked finalized selection and streamed I/O classification, +and the manual-recovery guard and exit classification. No blocking findings +remained. Persistent SQLite errors still reach terminal classification through +the new error wrapper; operational filesystem errors remain nonterminal. + +## Limits and landing work + +A repeated parallel host run reproduced the already registered +`dropped_runtime_scope_keeps_lock_until_detached_worker_stops` failure: final +reacquisition returned `Locked`. Its isolated rerun passed, and the complete +serial suite passed. This does not resolve the concurrency investigation; it +remains in the [review register](register.md#bounded-investigations-and-cleanup). + +The full 49-scenario rollups suite and Sepolia-image scenarios were not rerun. +Private-engine conformance/export and representative deployment capacity remain +in their existing integration plans. The two current recovery TLA+ models were +read but not changed or rerun; neither models these new boundary checks. + +No lower stack branch was rewritten. Reconcile the known C-host test ancestry +once at landing, retaining the evolved coverage at the reviewed tip and the +new independent post-recovery assertions. No push, PR creation, or merge was +performed during this implementation. diff --git a/docs/review/register.md b/docs/review/register.md index 238f2527..d9cf0b42 100644 --- a/docs/review/register.md +++ b/docs/review/register.md @@ -58,6 +58,8 @@ exposure in an actual deployment was established by this review. passes. The worker drops its scope before signalling completion, so a simple worker-completion race does not explain the failure. Identify any remaining descriptor/process ownership before changing the assertion or lock behavior. + Reproduced during the 2026-09-18 stack closeout; isolated and full serial + runs passed. See the [current validation record](2026-09-18-stack-review-validation.md). - **Transient SQLite contention stops the submitter.** Read handles use a 50 ms busy timeout; a storage/open failure escapes the submitter loop. BUSY/LOCKED are nonterminal but project to unclassified exit 1, causing diff --git a/docs/snapshots/lifecycle.md b/docs/snapshots/lifecycle.md index 0e9cd3f7..c190b528 100644 --- a/docs/snapshots/lifecycle.md +++ b/docs/snapshots/lifecycle.md @@ -44,9 +44,10 @@ Setup similarly makes the baseline artifact durable before publishing the complete era baseline, recovery root when applicable, and setup-completion facts atomically. -Restart selects the newest snapshot on the valid batch branch, or the baseline -when no batch snapshot exists. The selected artifact and stored application -count come from one row. Catch-up checks the restored engine's count and replays +Restart requires the newest valid closed batch's snapshot, or the baseline +when no valid closed batch exists. A missing required snapshot fails loud. +The selected artifact and stored application count come from one row. +Catch-up checks the restored engine's count and replays application inputs from that count. Invalidated branches are excluded by the same valid-batch relation used elsewhere. @@ -57,6 +58,12 @@ must exist; storage refuses a missing required row instead of falling back to an older snapshot. Acceptance already includes scheduler validation and local content identity, so merely observing an own-sender L1 input is insufficient. +A persisted canonical-divergence marker refuses accepted-checkpoint selection, +including when a matching batch precedes a divergent acceptance in the same +block. Selection checks the marker in the same SQLite transaction as its read +and any download lease. Finalized endpoints return HTTP 503 before consulting +conditional cache headers; they do not fall back to an older artifact. + This selection is independent of the lane's L1 reconciliation cursor. A crash between reader ingestion and lane reconciliation cannot miss a promotion or repeat one: acceptance is already durable, and the query derives the result. diff --git a/docs/threat-model/README.md b/docs/threat-model/README.md index 08df4af2..6db808c9 100644 --- a/docs/threat-model/README.md +++ b/docs/threat-model/README.md @@ -67,8 +67,10 @@ blocking production diagnostics would require revisiting that assumption batch/frame spine is re-inspected by the runtime danger detector within seconds of launch. The accepted residual is narrow: corrupt payload bytes in rows at/below the lane's resume checkpoint re-trip only when the WS - feed pages them (bounded by its catch-up window) or the submitter - re-encodes a pending batch, and a fault with no durable evidence (a panic + feed pages them or the submitter re-encodes a pending batch. The feed can + read the whole available history from era base `K` in bounded pages, with no + total catch-up limit; this detection path requires a subscriber to read the + affected rows. A fault with no durable evidence (a panic whose trigger does not recur) does not re-trip at all. The window is entered by restarting after a terminal exit (including supervisors that restart regardless of exit status), and @@ -76,7 +78,7 @@ blocking production diagnostics would require revisiting that assumption rollbackable soft confirmations, the watchdog byte-compare, and the I15 divergence freeze. - **Adversarial mempool:** reorder, delay, drop, selective inclusion by builders -- **Zombie transactions:** a submitted batch may sit in a private mempool indefinitely and land long after we believed it was gone. Two load-bearing defenses: the recovery flusher consumes every wallet-nonce slot this deployment ever used (anchored by the persisted watermark, I14) so zombies cannot claim them; and the content-identity check (I9/I15) compares every at/above-anchor *simulated-accepted* landing against the valid closed batch we sealed at that nonce. A foreign or byte-different landing records divergence when it becomes safe and is ingested, freezes the accepted frontier, and requires cockroach recovery. This is trust-boundary validation of external input (the mempool replaying our own stale transactions at times we don't control), not defense-in-depth against self-bugs or a general canonical-state oracle. In cockroach recovery the watermark does not survive the wipe, so that flush is best-effort by construction; the content-identity check is what keeps the residual zombie detected-and-frozen rather than silent (see [the flush boundary](../recovery/cockroach.md#flush-and-stopping-block)). +- **Zombie transactions:** a submitted batch may sit in a private mempool indefinitely and land long after we believed it was gone. Two load-bearing defenses: the recovery flusher consumes every wallet-nonce slot this deployment ever used (anchored by the persisted watermark, I14) so zombies cannot claim them; and the content-identity check (I9/I15) compares every *simulated-accepted* landing strictly after baseline block `C` against the valid closed batch we sealed at that nonce. The complete prefix through `C` is opaque and is never reinterpreted using the recovered nonce. A foreign or byte-different landing records divergence when it becomes safe and is ingested, freezes the accepted frontier, and requires cockroach recovery. This is trust-boundary validation of external input (the mempool replaying our own stale transactions at times we don't control), not defense-in-depth against self-bugs or a general canonical-state oracle. In cockroach recovery the watermark does not survive the wipe, so that flush is best-effort by construction; the content-identity check is what keeps the residual zombie detected-and-frozen rather than silent (see [the flush boundary](../recovery/cockroach.md#flush-and-stopping-block)). - L1 reorgs up to safe depth - Malicious `POST /tx` callers: malformed signatures, spoofed sender, replay across chains or apps, nonce manipulation - Malicious direct-input senders: arbitrary payload, any intent; sender authenticity is guaranteed by InputBox diff --git a/docs/watchdog/design-notes.md b/docs/watchdog/design-notes.md index 5c2e9ec8..c00b8ba0 100644 --- a/docs/watchdog/design-notes.md +++ b/docs/watchdog/design-notes.md @@ -44,16 +44,18 @@ accepted-batch wire-identity detector, and neither mechanism subsumes the other. The content-identity check runs inside the input reader's atomic -safe-input sync. For every -at/above-anchor landing the mirrored scheduler accepts, it requires a -byte-identical valid local sealed batch at that nonce. A foreign or mismatched +safe-input sync. For every landing strictly after baseline block `C` that +the mirrored scheduler accepts, it requires a byte-identical valid local sealed +batch at that nonce. A foreign or mismatched landing persists `canonical_divergence`, which freezes the accepted frontier -and the selection of newer accepted comparison checkpoints +and prevents accepted comparison checkpoint selection ([I15](../invariants.md#i15-divergence-marker-present--acceptance-frontier-frozen)). -The offending landing therefore normally never produces a newer -`/finalized_state/inclusion_block` for the watchdog to compare. Under the -unchanged-head optimization above, a watchdog tick legitimately exits idle. Distinct wire bytes can also be application-state -equivalent, which a byte comparison of resulting snapshots would not expose. +The prefix through `C` is opaque; historical landings are not reconsidered using +the recovered nonce. Once the marker is committed, finalized endpoints refuse +with HTTP 503, including when a matching batch preceded the divergent landing +in the same block. The watchdog cannot compare that block through these routes. +Distinct wire bytes can also be application-state equivalent, which a byte +comparison of resulting snapshots would not expose. Conversely, the content-identity check shares the sequencer's off-chain acceptance predicate and does not independently replay application execution. The watchdog can catch diff --git a/sequencer/src/storage/open.rs b/sequencer/src/storage/open.rs index 89c7d895..d9beba94 100644 --- a/sequencer/src/storage/open.rs +++ b/sequencer/src/storage/open.rs @@ -71,8 +71,8 @@ impl Storage { }) } - /// Create the schema and record its owning command in one migration - /// transaction. The complete history baseline is published later. On + /// Create the schema for setup or rebuild in one migration transaction. + /// The complete history baseline is published later. On /// an already-migrated database the hook does not run; callers must /// inspect the existing facts. pub(crate) fn initialize_for_command( From 24aa976c9e5dfa6a08d8a1bfb3139ba7f0a4ba45 Mon Sep 17 00:00:00 2001 From: gcdepaula Date: Sat, 19 Sep 2026 13:28:17 -0300 Subject: [PATCH 24/29] fix: preserve registered snapshots across path aliases --- docs/snapshots/lifecycle.md | 12 +- sequencer/src/commands/run/startup_hygiene.rs | 204 +++++++++++++++++- 2 files changed, 203 insertions(+), 13 deletions(-) diff --git a/docs/snapshots/lifecycle.md b/docs/snapshots/lifecycle.md index c190b528..c16902c8 100644 --- a/docs/snapshots/lifecycle.md +++ b/docs/snapshots/lifecycle.md @@ -135,8 +135,10 @@ of all checkpoint resources. Filesystem deletion failure leaves a harmless orphan for the startup sweep. The reverse ordering would leave a durable row pointing at missing state and is forbidden. Startup clears leases left by the dead process, checks the rollback checkpoint's `info.toml` and format version, collects obsolete -rows, and sweeps orphan directories before workers start. Application restoration -runs afterward in the launched inclusion lane, before processing new user ops; -the metadata check does not validate the application bytes. Missing or corrupt -referenced artifacts fail loud when read or restored; operational filesystem -errors retain their normal error classification. +rows, and sweeps orphan directories before workers start. The sweep resolves +every retained artifact path before deleting orphans and compares resolved paths +so alternate spellings and symlinks preserve the same artifact. Application +restoration runs afterward in the launched inclusion lane, before processing new +user ops; the metadata check does not validate the application bytes. Missing or +corrupt referenced artifacts fail loud when read or restored; operational +filesystem errors retain their normal error classification. diff --git a/sequencer/src/commands/run/startup_hygiene.rs b/sequencer/src/commands/run/startup_hygiene.rs index 0271aeb4..b01136c2 100644 --- a/sequencer/src/commands/run/startup_hygiene.rs +++ b/sequencer/src/commands/run/startup_hygiene.rs @@ -61,24 +61,38 @@ fn snapshot_gc_at_startup(storage: &mut crate::storage::Storage) -> Result Result { - let known: std::collections::HashSet = storage + let known = storage .list_dump_rows()? .into_iter() - .map(|row| row.prefix) - .collect(); + .map(|row| { + std::fs::canonicalize(&row.prefix).map_err(|source| { + CommandError::ReferencedSnapshotArtifact { + path: row.prefix, + source, + } + }) + }) + .collect::, _>>()?; let mut removed = 0; for entry in std::fs::read_dir(dumps_dir)? { let entry = entry?; let path = entry.path(); - if known.contains(&path) { + let retained = match std::fs::canonicalize(&path) { + Ok(resolved) => known.contains(&resolved), + // GC or an earlier orphan deletion can leave an unregistered + // dangling symlink. Retained references already resolved above. + Err(err) if err.kind() == std::io::ErrorKind::NotFound => false, + Err(err) => return Err(err.into()), + }; + if retained { continue; } match delete_dump_dir(&path) { @@ -183,6 +197,180 @@ mod tests { assert_eq!(removed, 0); } + #[test] + fn sweep_preserves_mixed_references_across_path_spellings() { + for spelling in [ + "relative-to-absolute", + "absolute-to-relative", + "stored-dot", + "sweep-dot", + ] { + let db = temp_db(spelling); + let mut storage = Storage::open(&db.path).unwrap(); + // Avoid changing the process-wide cwd while other tests are running. + let root = tempfile::tempdir_in(".").unwrap(); + let relative = std::path::PathBuf::from(root.path().file_name().unwrap()).join("dumps"); + std::fs::create_dir(&relative).unwrap(); + let absolute = relative.canonicalize().unwrap(); + let (stored_dir, sweep_dir) = match spelling { + "relative-to-absolute" => (relative.clone(), absolute.clone()), + "absolute-to-relative" => (absolute.clone(), relative), + "stored-dot" => (std::path::Path::new(".").join(&relative), relative), + "sweep-dot" => (relative.clone(), std::path::Path::new(".").join(relative)), + _ => unreachable!(), + }; + let tracked = absolute.join("tracked"); + create_structured_dump(&tracked); + storage + .insert_baseline_snapshot( + &stored_dir.join("tracked"), + crate::storage::ExecutedInputCount::ZERO, + ) + .unwrap(); + // A literal match must not hide the other row's aliased spelling. + let literal_match = sweep_dir.join("literal-match"); + create_structured_dump(&literal_match); + storage + .write(|tx| { + tx.execute( + "INSERT INTO dumps(prefix) VALUES (?1)", + [literal_match.to_str().unwrap()], + ) + }) + .unwrap(); + let orphan = absolute.join("orphan"); + create_structured_dump(&orphan); + + assert_eq!( + sweep_orphan_dumps(&mut storage, &sweep_dir).unwrap(), + 1, + "{spelling}" + ); + + assert!(tracked.join("info.toml").is_file(), "{spelling}"); + assert!(literal_match.join("info.toml").is_file(), "{spelling}"); + assert!(!orphan.exists(), "{spelling}"); + assert_eq!(storage.list_dump_rows().unwrap().len(), 2); + } + } + + #[cfg(unix)] + #[test] + fn sweep_preserves_symlinked_parent_and_dump_aliases() { + for stored_via_alias in [false, true] { + let db = temp_db("sweep-symlinks"); + let mut storage = Storage::open(&db.path).unwrap(); + let root = tempfile::tempdir().unwrap(); + let dumps = root.path().join("dumps"); + std::fs::create_dir(&dumps).unwrap(); + let parent_alias = root.path().join("parent-alias"); + std::os::unix::fs::symlink(&dumps, &parent_alias).unwrap(); + let tracked = dumps.join("tracked"); + create_structured_dump(&tracked); + let dump_alias = dumps.join("dump-alias"); + std::os::unix::fs::symlink(&tracked, &dump_alias).unwrap(); + let stored = if stored_via_alias { + parent_alias.join("dump-alias") + } else { + tracked.clone() + }; + storage + .insert_baseline_snapshot(&stored, crate::storage::ExecutedInputCount::ZERO) + .unwrap(); + let orphan = dumps.join("orphan"); + create_structured_dump(&orphan); + let sweep_dir = if stored_via_alias { + &dumps + } else { + &parent_alias + }; + + assert_eq!(sweep_orphan_dumps(&mut storage, sweep_dir).unwrap(), 1); + + assert!(tracked.join("info.toml").is_file()); + assert!(dump_alias.join("info.toml").is_file()); + assert!(stored.join("info.toml").is_file()); + assert!(!orphan.exists()); + } + } + + #[test] + fn sweep_resolves_every_reference_before_deleting_any_artifact() { + let db = temp_db("sweep-unresolved-reference"); + let mut storage = Storage::open(&db.path).unwrap(); + let dumps = tempfile::tempdir().unwrap(); + let tracked = dumps.path().join("tracked"); + create_structured_dump(&tracked); + storage + .insert_baseline_snapshot( + &dumps.path().join(".").join("tracked"), + crate::storage::ExecutedInputCount::ZERO, + ) + .unwrap(); + let missing = dumps.path().join("missing"); + storage + .write(|tx| { + tx.execute( + "INSERT INTO dumps(prefix) VALUES (?1)", + [missing.to_str().unwrap()], + ) + }) + .unwrap(); + let orphan = dumps.path().join("orphan"); + create_structured_dump(&orphan); + + let error = sweep_orphan_dumps(&mut storage, dumps.path()).unwrap_err(); + + assert!(matches!( + &error, + CommandError::ReferencedSnapshotArtifact { path, source } + if path == &missing && source.kind() == std::io::ErrorKind::NotFound + )); + assert_eq!(error.exit_code(), crate::commands::error::EXIT_TERMINAL); + assert!(tracked.join("info.toml").is_file()); + assert!( + orphan.join("info.toml").is_file(), + "resolution precedes deletion" + ); + assert_eq!(storage.list_dump_rows().unwrap().len(), 2); + } + + #[cfg(unix)] + #[test] + fn sweep_removes_orphan_symlinks_without_following_them() { + let db = temp_db("sweep-orphan-symlinks"); + let mut storage = Storage::open(&db.path).unwrap(); + let root = tempfile::tempdir().unwrap(); + let dumps = root.path().join("dumps"); + std::fs::create_dir(&dumps).unwrap(); + let tracked = dumps.join("tracked"); + create_structured_dump(&tracked); + storage + .insert_baseline_snapshot(&tracked, crate::storage::ExecutedInputCount::ZERO) + .unwrap(); + let outside = root.path().join("outside"); + create_structured_dump(&outside); + let orphan = dumps.join("orphan"); + create_structured_dump(&orphan); + for (name, target) in [ + ("orphan-alias", orphan), + ("dangling", root.path().join("missing")), + ("outside-alias", outside.clone()), + ] { + std::os::unix::fs::symlink(target, dumps.join(name)).unwrap(); + } + + assert_eq!(sweep_orphan_dumps(&mut storage, &dumps).unwrap(), 4); + assert!(tracked.join("info.toml").is_file()); + assert!(outside.join("info.toml").is_file()); + let remaining: Vec<_> = std::fs::read_dir(&dumps) + .unwrap() + .map(|entry| entry.unwrap().file_name()) + .collect(); + assert_eq!(remaining, vec![std::ffi::OsString::from("tracked")]); + assert_eq!(storage.list_dump_rows().unwrap().len(), 1); + } + #[test] fn startup_resets_persisted_crash_leases_before_collecting_artifacts() { let db = temp_db("startup-crash-leases"); From c22174a667d9c79be26bf721307af3274795fa17 Mon Sep 17 00:00:00 2001 From: gcdepaula Date: Sat, 19 Sep 2026 13:28:41 -0300 Subject: [PATCH 25/29] docs: require complete pending-direct recovery checkpoints --- docs/plans/2026-07-coordination-tracks.md | 5 +- docs/protocol/application-contract.md | 4 ++ docs/protocol/projection-replay.md | 7 +- docs/recovery/cockroach.md | 43 +++++++++--- sequencer-core/src/scheduler/fold.rs | 81 ++++++++++++++++++++++- sequencer/src/commands/setup/mod.rs | 4 +- 6 files changed, 130 insertions(+), 14 deletions(-) diff --git a/docs/plans/2026-07-coordination-tracks.md b/docs/plans/2026-07-coordination-tracks.md index 75c8581b..7f591d78 100644 --- a/docs/plans/2026-07-coordination-tracks.md +++ b/docs/plans/2026-07-coordination-tracks.md @@ -76,7 +76,10 @@ Remaining checks need the actual consumer: for production readiness: the old native state is unavailable, the exported bundle restores correctly, and execution after rebuild matches the canonical machine. For the DEX, pin the designated state drive/memory region and derive - resume metadata from canonical execution. Add the integration check to the + resume metadata from canonical execution. The exporter must check + [pending-direct eligibility](../recovery/cockroach.md#checkpoint-eligibility) + and the drill must exercise refusal and earlier-checkpoint fallback, alongside + eligible pending-queue recovery. Add the integration check to the release validation once the actual artifacts are available; no generic trait or deployment gate currently enforces this requirement. - Repeat snapshot-to-live bootstrap and canonical comparison with the private diff --git a/docs/protocol/application-contract.md b/docs/protocol/application-contract.md index 3bc28a28..24a45461 100644 --- a/docs/protocol/application-contract.md +++ b/docs/protocol/application-contract.md @@ -209,6 +209,10 @@ layout, and extraction procedure for each supported image. Other applications may require a different mapping. The recovery bundle also needs the exact L1 boundary and next scheduler nonce, obtained from trusted canonical execution; these are separate from merely extracting application bytes. +The exporter must also verify the canonical pending-direct queue satisfies +[checkpoint eligibility](../recovery/cockroach.md#checkpoint-eligibility). +An accurate application clock and `A < B` do not prove that condition after +faulty sequencing. Each integration supplies a versioned export command and operator procedure, and demonstrates recovery from a non-genesis canonical checkpoint before diff --git a/docs/protocol/projection-replay.md b/docs/protocol/projection-replay.md index 33f582f6..7278c485 100644 --- a/docs/protocol/projection-replay.md +++ b/docs/protocol/projection-replay.md @@ -66,8 +66,11 @@ share a count, and generations can reuse replaced offsets. Such a backup contains core state and projection at count `X`, inclusion block `B`, next scheduler nonce `N`, and the application's own clock `A`. The -[manual recovery contract](../recovery/cockroach.md#replay-boundaries) requires -`A < B`, except known empty genesis, and `B <= C` for the target rebuild: +[manual recovery contract](../recovery/cockroach.md#checkpoint-eligibility) +requires no pending canonical direct at or below `A`, as well as `A < B` +(except known empty genesis) and `B <= C` for the target rebuild. Establish +queue eligibility against the canonical checkpoint; application-state equality +and the scalar bounds alone cannot prove it after faulty sequencing: 1. Independently establish trust in the backup, projection implementation, and checkpoint boundary under the [incident playbook](../recovery/cockroach.md#application-specific-reader-state). diff --git a/docs/recovery/cockroach.md b/docs/recovery/cockroach.md index 86aff380..72bbab51 100644 --- a/docs/recovery/cockroach.md +++ b/docs/recovery/cockroach.md @@ -45,7 +45,8 @@ The application runbook must name: - How to select and preserve a trusted CM checkpoint and its exact L1 boundary. - The command extracting the application state, count/clock, and next scheduler nonce into the bundle accepted by `setup --recovery`, including supported - checkpoint boundaries and the loader's `A < B` requirement below. + checkpoint boundaries and verification of the + [pending-direct eligibility condition](#checkpoint-eligibility) below. - Artifact locations, access/backup procedures, validation commands, and the commands to rebuild, restart, compare, and resume affected readers. - A rehearsed fallback to an earlier trusted checkpoint or genesis if the @@ -56,6 +57,9 @@ pinned deployment data, and L1 access, while the old native database and dumps are unavailable. Run the actual exporter and restore its output; check the native application bytes/progress and scheduler nonce against the canonical source. Exercise directs pending at the checkpoint and inputs arriving after it. +The exporter must refuse a checkpoint with a pending direct at or below its +application clock, even when `A < B`; rehearse the earlier-checkpoint/genesis +fallback. Also cover an eligible nonempty queue whose directs are all above `A`. Run `setup --recovery` in a fresh directory, resume sequencing, and compare against independent canonical execution after a new batch is accepted. The terminal-drained baseline itself need not equal canonical state at `C`. @@ -95,9 +99,11 @@ latest possible sound checkpoint is not a prerequisite for recovery. state without establishing that all earlier executions were correct. 4. **Prepare a restorable native bundle.** Validate the candidate's restored application state, embedded count/clock, and next batch nonce against the - canonical reference at block `B`. Keep the artifact and its boundary metadata - together and record how its trust was established. The receipt alone is not - evidence that a faulty sequencer executed correctly. + canonical reference at block `B`. Verify its pending-direct queue meets + [checkpoint eligibility](#checkpoint-eligibility); otherwise choose an + eligible earlier checkpoint or genesis. Keep the artifact and its boundary + metadata together and record how its trust was established. The receipt alone + is not evidence that a faulty sequencer executed correctly. 5. **Rebuild in a fresh directory.** Use the invocation below. Recovery chooses its post-flush stopping block `C`, replays from the trusted checkpoint, and publishes a new era. Preserve that baseline artifact and its metadata for @@ -191,12 +197,33 @@ The replay boundaries are: | `S'`, `N'` | Recovered state and next batch nonce. | | `K` | Application count in `S'`; the first later application input has offset `K`. | +### Checkpoint eligibility + +At the exact end-of-block boundary `B`, the canonical scheduler must have **no +pending direct with inclusion block `<= A`**. The checkpoint therefore already +accounts for every direct through `A`; all remaining directs are reconstructed +from `(A, B]`. The exporter must inspect the canonical queue to establish this +condition, including when validating a retained native artifact against the CM. +If it fails, refuse the export and select an eligible earlier checkpoint or +genesis. + +Honest live sequencing establishes this condition: a frame's safe block precedes +its L1 inclusion, so all directs it covers have already arrived. The canonical +scheduler also accepts equality, however. After faulty sequencing, a batch at +block 10 can execute at clock 10 before another direct in that block arrives. +An empty batch at block 11 advances the nonce without draining that direct. +The truthful checkpoint has `A=10 < B=11`, yet seeding `(A,B]` would omit it. +Application-state equality and `A < B` alone cannot establish eligibility. + Loading checks the receipt's block against configured `B` and its nonce against `info.toml`. It requires `A < B`, except for known empty genesis (`B`, nonce, and application count all zero). At non-genesis `A = B`, a direct arriving after the accepted batch in block `B` could still be pending but disappear from the seed -range. Checkpoint state and nonce remain operator-trusted; the later -content-identity check does not verify this prefix. +range. The bundle contains no scheduler queue evidence, so the loader cannot +verify the pending-direct condition. Eligibility, checkpoint state, and nonce +remain operator-trusted; the later content-identity check does not verify this +prefix. Exporter enforcement belongs to the application's required recovery +integration, not the generic loader. The complete ordering is `A < B <= C`, with `A = B = 0` allowed for empty genesis. Before sourcing or executing the fold, recovery requires `B <= C`. @@ -227,8 +254,8 @@ retried after the node catches up. Seed the scheduler's pending-direct queue from `(A, B]`, excluding inputs sent by the batch submitter. Then replay **all raw inputs** in `(B, C]` in L1 order with expected nonce `N`. Drain the remaining directs through `C` to obtain `(S', N')`. -The disjoint ranges preserve pending directs without executing the checkpoint's -accepted batches again. +For an [eligible checkpoint](#checkpoint-eligibility), the disjoint ranges +preserve pending directs without executing its accepted batches again. On the first `run` sync, acceptance starts at nonce `N'` and scans only blocks **strictly after `C`**. Nonce filtering alone would let a previously rejected diff --git a/sequencer-core/src/scheduler/fold.rs b/sequencer-core/src/scheduler/fold.rs index de340efc..0448a502 100644 --- a/sequencer-core/src/scheduler/fold.rs +++ b/sequencer-core/src/scheduler/fold.rs @@ -44,8 +44,10 @@ pub struct FoldInput { /// `B` and its scheduler nonce `N` (metadata — the bare-metal app cannot /// recompute it, so the engine is *told* it via `resume_at`). /// - `seeds`: directs reconstructed from `(A, B]`, with sequencer-sourced -/// batches already dropped by the caller (their content is already in `S`, -/// their frames' safe blocks `≤ A`). Must arrive in ascending L1 order. +/// batches already dropped by the caller. The checkpoint must account for +/// every direct through its application clock `A`: the exporter verifies no +/// pending canonical direct has inclusion block `<= A`. `A < B` alone cannot +/// establish this after faulty sequencing. Seeds arrive in ascending L1 order. /// - `replay`: the full `(B, C]` stream (batches + directs) in L1 order. The /// scheduler classifies each input (batch iff `sender == sequencer_address`), /// force-executes overdue directs on arrival, applies accepted batches, @@ -517,6 +519,81 @@ mod tests { ); } + #[test] + fn earlier_checkpoint_recovers_a_direct_hidden_below_a_later_checkpoint_clock() { + let feed = |scheduler: &mut Scheduler, inputs: Vec| { + for input in inputs { + scheduler + .process_input(SchedulerInput { + sender: input.sender, + inclusion_block: input.inclusion_block, + domain: domain(), + payload: input.payload, + }) + .expect("canonical execution"); + } + }; + let mut canonical = Scheduler::new(FoldApp::default(), config()); + feed( + &mut canonical, + vec![ + direct(DIRECT_SENDER, 5, 1), + cover_batch(9, 0, 5), + direct(DIRECT_SENDER, 9, 2), + ], + ); + + let checkpoint = canonical.app.clone(); + let checkpoint_nonce = canonical.next_expected_batch_nonce(); + let a = checkpoint.last_executed_safe_block(); + assert_eq!(a, 5); + assert!( + canonical + .direct_q + .iter() + .all(|input| input.inclusion_block > a), + "the earlier checkpoint is eligible, with a nonempty seed queue" + ); + assert_eq!(canonical.queued_direct_len(), 1); + + // Equality is canonically valid even though honest live sequencing + // cannot submit a batch into its own already-safe block. + let replay = vec![ + direct(DIRECT_SENDER, 10, 3), + cover_batch(10, 1, 10), + direct(DIRECT_SENDER, 10, 4), + empty_batch(11, 2), + ]; + feed(&mut canonical, replay.clone()); + let later_a = canonical.app.last_executed_safe_block(); + assert_eq!(later_a, 10); + assert!(later_a < 11, "the later checkpoint passes the scalar bound"); + assert_eq!(canonical.next_expected_batch_nonce(), 3); + assert_eq!(canonical.app.executed_directs, vec![1, 2, 3]); + assert_eq!(canonical.queued_direct_len(), 1); + assert_eq!(canonical.direct_q[0].payload, vec![4]); + assert_eq!(canonical.direct_q[0].inclusion_block, later_a); + // Its pending direct is excluded by (A,B], so the exporter must refuse + // this later checkpoint. Recovery can use the eligible earlier one. + + canonical.drain_covered_at(11).expect("terminal drain"); + let (expected, expected_nonce) = canonical.finish(); + let (recovered, recovered_nonce) = fold_replay( + checkpoint, + checkpoint_nonce, + config(), + domain(), + vec![direct(DIRECT_SENDER, 9, 2)], + replay, + 11, + ) + .expect("recovery from eligible checkpoint"); + assert_eq!(recovered.executed_directs, vec![1, 2, 3, 4]); + assert_eq!(recovered.executed_directs, expected.executed_directs); + assert_eq!(recovered.progress(), expected.progress()); + assert_eq!(recovered_nonce, expected_nonce); + } + #[test] fn fold_replay_reconstructs_identical_state_to_a_live_run() { // Differential: splitting the (genesis, C] stream at an intermediate B diff --git a/sequencer/src/commands/setup/mod.rs b/sequencer/src/commands/setup/mod.rs index df2dea7c..69ecb0a0 100644 --- a/sequencer/src/commands/setup/mod.rs +++ b/sequencer/src/commands/setup/mod.rs @@ -423,6 +423,8 @@ impl Checkpoint { /// Load `S` from the dump dir, derive `A` and `N`, and enforce the load-time /// precondition `A < B`, except for the known empty genesis checkpoint. /// Equality otherwise hides pending directs later in the same block. + /// The exporter must separately verify no canonical direct at or below A + /// remains queued; this bundle carries no queue evidence to check here. fn load(dir: &std::path::Path, checkpoint_block: u64) -> Result { let load_err = |message: String| SetupRecoveryError::CheckpointLoad { path: dir.display().to_string(), @@ -503,7 +505,7 @@ fn source_fold_inputs( /// L1 failures leave setup incomplete for a fresh attempt. /// /// The `flush → fold → fill` steps are enumerated authoritatively in -/// **[`docs/recovery/cockroach.md`](../../../docs/recovery/cockroach.md)** (spec, +/// **[`docs/recovery/cockroach.md`](../../../../docs/recovery/cockroach.md)** (spec, /// data dictionary `A`/`B`/`C`/`N`/`N'`, and code map) and anchored inline below /// (`// 1.`…`// 6.`). Read the doc before editing this function. /// From dc4dd782777ac4d52f1743fd8867bfeb62c74c1f Mon Sep 17 00:00:00 2001 From: gcdepaula Date: Sat, 19 Sep 2026 13:29:44 -0300 Subject: [PATCH 26/29] test: pin recovered-prefix cutoff and clarify API contracts --- README.md | 20 ++ docs/invariants.md | 2 +- .../2026-09-19-review-followup-validation.md | 65 ++++++ docs/watchdog/operator-deployment.md | 28 +-- sequencer/src/l1/submitter/poster.rs | 3 + sequencer/src/recovery/mod.rs | 8 +- .../src/storage/safe_accepted_batches.rs | 219 +++++++++--------- 7 files changed, 221 insertions(+), 124 deletions(-) create mode 100644 docs/review/2026-09-19-review-followup-validation.md diff --git a/README.md b/README.md index 1e791634..ee68b2f6 100644 --- a/README.md +++ b/README.md @@ -234,6 +234,8 @@ After each successfully applied input at offset `X`, persist the claim with `HISTORY_UNAVAILABLE`, or `AHEAD_OF_HEAD`. Rebootstrap on a history mismatch. - A claim exactly at the head waits for the next input. Replay uses bounded pages and queues, with no total catch-up limit. The subscriber cap is `64`. +- Before upgrade, capacity exhaustion returns `429 OVERLOADED`; shutdown or an + operational subscription failure returns `503 UNAVAILABLE`. - Messages are JSON text frames; binary fields are `0x`-prefixed hex. Direct-input `block_timestamp` values are Unix seconds. - Batch envelopes are absent. Offsets count executed application inputs, @@ -261,6 +263,11 @@ L1 and then join the application feed. The [projection replay contract](docs/protocol/projection-replay.md) describes bootstrap, client checkpoints, pending directs, and terminal drain. +WS `sender` strings use EIP-55 checksum casing; address fields in `/history` +and `sender` strings in `/historical-l1-inputs` use lowercase hex. Clients must +compare decoded 20-byte addresses and use one normalized encoding for projection +keys across these feeds. + `GET /history` returns one coherent view of the deployment, current application history, immutable era baseline, and latest accepted checkpoint. Optional `era_id=` requires the selected era; a mismatch returns `409 ERA_CHANGED`. @@ -411,6 +418,19 @@ all three finalized endpoints return `503 UNAVAILABLE`, including conditional state requests. The check shares the checkpoint-selection transaction, before any lease or archive is created. See [snapshot lifecycle](docs/snapshots/lifecycle.md). +### Health probes (internal only) + +- `GET /livez` returns `200` whenever the handler responds, with an empty body. +- `GET /readyz` returns `200` while the inclusion-lane receiver is open and + shutdown has not been requested; otherwise `503`. Its body is empty. +- `GET /healthz` uses the same status as `/readyz` and returns JSON: + `{ "status": "ok", "inclusion_lane": "ok" }`. `status` becomes `"degraded"` + for either failure condition; `inclusion_lane` becomes `"stopped"` only when + its receiver is closed, so it can remain `"ok"` during shutdown. + +These probes cover process reachability, the lane channel, and shutdown state. +They do not certify L1 freshness, submitter balance, or canonical agreement. + ## Storage Model - `batches`: batch metadata diff --git a/docs/invariants.md b/docs/invariants.md index 3600002e..0a68a8d9 100644 --- a/docs/invariants.md +++ b/docs/invariants.md @@ -80,7 +80,7 @@ by writer and are write-once (`0001_schema.sql`). |---|---| | inclusion lane | `batches` (insert + `sealed_at_ms`), `frames`, `user_ops`, `application_inputs`, `dumps`/`snapshots` (batch close) | | input reader | `safe_inputs`, `l1_safe_head`, `safe_accepted_batches`, `canonical_divergence` (the divergence poison marker) | -| recovery (startup) | `batches.invalidated_at_ms`, Tip reopen, current `application_inputs` suffix deletion | +| recovery (startup) | `batches.invalidated_at_ms`, Tip reopen, current `application_inputs` suffix deletion and replacement direct-input rows | | history metadata (setup/recovery) | `history_state` — complete era/application-count/L1-block baseline and generation; `history_generation_cuts` — immutable preserved-prefix cuts written with non-empty standard-recovery cascades | | batch submitter and mempool flusher | `wallet_nonce_watermark` — deliberately shared under one protocol: each raises it before its first broadcast (write-before-broadcast, I14) | | egress (HTTP) | `dumps.lease_count` (leases); `run`'s startup hygiene resets it to zero as the crash backstop | diff --git a/docs/review/2026-09-19-review-followup-validation.md b/docs/review/2026-09-19-review-followup-validation.md new file mode 100644 index 00000000..68380426 --- /dev/null +++ b/docs/review/2026-09-19-review-followup-validation.md @@ -0,0 +1,65 @@ +# Review follow-up validation + +Evidence for the ongoing stack review and landing, covering changes above +`f504c2e88e2b432718493e29e3c56f74fce3ad00` on `codex/stack-review-fixes`. +Retire this record after landing when no ongoing review decision uses it. + +## Changes and discriminating checks + +- Startup compares resolved artifact paths after resolving every retained + reference. Regressions preserve relative/absolute, leading-dot, and symlink + aliases, including mixed literal/aliased references, while removing genuine + orphans. A missing reference stops the sweep before any deletion. Dangling + orphan links are removed without following their targets. All ten startup + hygiene tests pass. +- A scheduler/fold regression establishes that `A < B` can coexist with a + pending direct at `A` after a same-block frame and a later empty batch. It + verifies recovery from an eligible earlier checkpoint preserves every direct, + application progress, and the scheduler nonce. The + [checkpoint contract](../recovery/cockroach.md#checkpoint-eligibility) requires + canonical exporters to inspect the pending queue and refuse such candidates. +- The accepted-prefix fixture now places the old future-nonce input both below + and exactly at `C`, before the nonce-0 batch in L1 order. Temporarily changing + the production query from `> C` to `>= C` fails specifically at the equality + case. Restoring `>` passes all four acceptance-projection tests. +- API documentation states the existing address-casing/normalization contract, + WS admission statuses, health-probe semantics, and internal deployment + boundary. Writer ownership and stale source references were corrected. The + submitter's own-sender decode error remains visible under self-trust. + +Separate reviewers examined snapshot cleanup and checkpoint eligibility. Review +caught the dangling-orphan-link case before final validation; its regression is +included. No findings remain within the implemented scope. + +## Validation + +macOS arm64, parent Nix/direnv environment, Rust and Cargo 1.95.0: + +| Check | Result | +|---|---| +| `cargo test --locked --workspace --exclude canonical-test -- --test-threads=1` | 759 passed; zero failed; one existing ignored harness doc test | +| `cargo check --locked --workspace --all-targets` | Passed | +| `cargo clippy --locked --workspace --all-targets --all-features -- -D warnings` | Passed | +| `cargo fmt --all -- --check` and `git diff --check` | Passed | +| Relative Markdown links in the changed contract/API documents | 111 targets/anchors checked | + +The full host suite includes the new regressions. Guest execution, standalone +rollups E2Es, watchdog Lua tests, and TLA+ model runs were not repeated for this +follow-up. Both current recovery models were read; their modeled transitions +are unchanged. Earlier validation remains in the +[initial closeout record](2026-09-18-stack-review-validation.md). + +## Remaining integration and landing work + +The bundle carries no canonical queue evidence, and no canonical-to-native +exporter is implemented here. The eligibility change is a supported-checkpoint +precondition and executable scheduler regression, not loader enforcement or a +completed CM export drill. The application's exporter must enforce refusal and +demonstrate fallback and eligible pending-queue recovery; the +[integration plan](../plans/2026-07-coordination-tracks.md#track-6--dump--application-api-redesign) +tracks that work. Canonical acceptance semantics and artifact formats are unchanged. + +Archive concurrency limits remain deferred pending deployment workload needs. +The existing process-lock concurrency investigation remains open; this host run +used the serial suite. The lower-stack C-host ancestry reconciliation remains +landing work. No remote branch, PR discussion, or merge was changed. diff --git a/docs/watchdog/operator-deployment.md b/docs/watchdog/operator-deployment.md index e7c3c7d4..10b8082e 100644 --- a/docs/watchdog/operator-deployment.md +++ b/docs/watchdog/operator-deployment.md @@ -9,20 +9,20 @@ For **local development only** (Anvil + `sequencer-devnet`, CI smoke tests), use ## Two deployment tiers ```text - ┌──────────────────────────────────────────────┐ - Internet / users │ Public ingress (POST /tx, GET /fee, WS) │ ← benchmarks, wallets - └──────────────────────┬───────────────────────┘ - │ - ┌─────────────────▼───────────────────┐ - Operator network │ Sequencer process │ - │ + internal snapshot HTTP │ ← watchdog ONLY here - │ /finalized_state* │ - └─────────┬───────────────┬─────────┘ - │ │ - ┌─────────▼───┐ ┌───────▼────────┐ - │ L1 (Sepolia │ │ Watchdog host │ - │ or mainnet)│ │ (compare) │ - └─────────────┘ └────────────────┘ + ┌─────────────────────────────────────┐ + Internet / users │ Public ingress: POST /tx, GET /fee │ ← wallets + └──────────────────┬──────────────────┘ + │ + ┌──────────────────▼──────────────────┐ + Operator network │ Sequencer process │ + │ Internal WS, history, and health │ ← indexers and probes + │ Snapshots: /finalized_state* │ ← watchdog + └─────────┬─────────────────┬─────────┘ + │ │ + ┌─────────▼────────┐ ┌──────▼─────────┐ + │ L1 (Sepolia │ │ Watchdog host │ + │ or mainnet) │ │ (compare) │ + └──────────────────┘ └────────────────┘ ``` The watchdog independently replays L1 through the canonical CM and compares diff --git a/sequencer/src/l1/submitter/poster.rs b/sequencer/src/l1/submitter/poster.rs index aa6dc904..6db3ef6f 100644 --- a/sequencer/src/l1/submitter/poster.rs +++ b/sequencer/src/l1/submitter/poster.rs @@ -532,6 +532,9 @@ impl BatchPoster for EthereumBatchPoster { if evm_advance.msgSender != self.config.batch_submitter_address { continue; } + // This dedicated key emits our own well-formed batches. A decode + // failure is evidence to investigate under self-trust, not an + // untrusted direct input to skip (docs/threat-model/README.md). let batch: Batch = ssz::Decode::from_ssz_bytes(evm_advance.payload.as_ref()) .map_err(|err| BatchPosterError::Provider(format!("{err:?}")))?; observed_nonces.push(batch.nonce); diff --git a/sequencer/src/recovery/mod.rs b/sequencer/src/recovery/mod.rs index 48bb046f..3bf9b9f9 100644 --- a/sequencer/src/recovery/mod.rs +++ b/sequencer/src/recovery/mod.rs @@ -438,10 +438,10 @@ fn classify_signer_provider( /// build), refused because re-running the same boot re-fails /// identically. The discovery-time facts (wrong contract, pre-v3 /// InputBox) never reach this function: they arise in `InputReader::new`, -/// which only `setup` calls and projects as a worker exit (register -/// finding 32). In the live loop the same URL was already proven by this -/// boot's initial sync, so a live `Bootstrap` restarts unclassified -/// rather than poisoning the data directory. +/// which only `setup` calls and currently projects as a live-worker exit +/// rather than a deterministic configuration failure. In the live loop the +/// same URL was already proven by this boot's initial sync, so a live +/// `Bootstrap` restarts unclassified rather than poisoning the data directory. /// - `Join` (a non-panic loss of a storage task) is shutdown-path /// cancellation in the live loop. During startup the runtime that would /// cancel it is the one driving this boot, so an unexplained loss is diff --git a/sequencer/src/storage/safe_accepted_batches.rs b/sequencer/src/storage/safe_accepted_batches.rs index 7d37fffd..3294b13c 100644 --- a/sequencer/src/storage/safe_accepted_batches.rs +++ b/sequencer/src/storage/safe_accepted_batches.rs @@ -387,115 +387,124 @@ mod tests { use sequencer_core::{batch::Batch, history::ExecutedInputCount}; use ssz::Encode; - let db = temp_db("accepted-recovered-prefix"); - let mut storage = Storage::initialize_for_command(&db.path, LifecycleCommand::Rebuild) - .expect("initialize rebuild"); - let submitter = Address::repeat_byte(0x99); - let timing = default_protocol_timing(); - pin_test_deployment_identity(&mut storage, submitter); - let old_future = Batch { - nonce: 1, - frames: vec![], - } - .as_ssz_bytes(); - let old_accepted = Batch { - nonce: 0, - frames: vec![], - } - .as_ssz_bytes(); - assert!( - timing - .scheduler_accepts( - submitter, - SafeInputView { - safe_input_index: 0, - sender: submitter, - payload: &old_future, - inclusion_block: 20, - }, - 0 - ) - .is_none() - ); - assert!( - timing - .scheduler_accepts( + // At C itself, the future nonce must precede nonce 0 in L1 order: + // it was rejected there and must not become accepted after the rebuild. + for old_future_block in [20, 30] { + let db = temp_db("accepted-recovered-prefix"); + let mut storage = Storage::initialize_for_command(&db.path, LifecycleCommand::Rebuild) + .expect("initialize rebuild"); + let submitter = Address::repeat_byte(0x99); + let timing = default_protocol_timing(); + pin_test_deployment_identity(&mut storage, submitter); + let old_future = Batch { + nonce: 1, + frames: vec![], + } + .as_ssz_bytes(); + let old_accepted = Batch { + nonce: 0, + frames: vec![], + } + .as_ssz_bytes(); + assert!( + timing + .scheduler_accepts( + submitter, + SafeInputView { + safe_input_index: 0, + sender: submitter, + payload: &old_future, + inclusion_block: old_future_block, + }, + 0 + ) + .is_none() + ); + assert!( + timing + .scheduler_accepts( + submitter, + SafeInputView { + safe_input_index: 1, + sender: submitter, + payload: &old_accepted, + inclusion_block: 30, + }, + 0 + ) + .is_some() + ); + storage + .append_safe_inputs_with_timestamp( + 30, + 30, + &[ + StoredSafeInput { + sender: submitter, + payload: old_future, + block_number: old_future_block, + }, + StoredSafeInput { + sender: submitter, + payload: old_accepted, + block_number: 30, + }, + ], submitter, - SafeInputView { - safe_input_index: 1, - sender: submitter, - payload: &old_accepted, - inclusion_block: 30, - }, - 0 + &timing, + FrontierMode::DeferUntilAnchorSet, ) - .is_some() - ); - storage - .append_safe_inputs_with_timestamp( - 30, - 30, - &[ - StoredSafeInput { - sender: submitter, - payload: old_future, - block_number: 20, - }, - StoredSafeInput { - sender: submitter, - payload: old_accepted, - block_number: 30, - }, - ], - submitter, - &timing, - FrontierMode::DeferUntilAnchorSet, - ) - .expect("ingest opaque prefix"); - storage - .write(|tx| { - super::super::history::initialize_history_in(tx, ExecutedInputCount::new(41), 30)?; - super::super::mutations::set_batch_tree_anchor_in(tx, 1)?; - super::super::ingress::open_recovery_tip_in_tx(tx, 30) - }) - .expect("install recovered baseline"); - let mut head = storage.open_state().expect("read root").expect("root"); - storage - .close_frame_and_batch(&mut head, 30) - .expect("close resumed batch"); - let payload = local_batch_payload(&mut storage, 1); - storage - .append_safe_inputs( - 31, - &[StoredSafeInput { - sender: submitter, - payload, - block_number: 31, - }], - submitter, - &timing, - ) - .expect("accept post-baseline batch"); - assert!( + .expect("ingest opaque prefix"); storage - .canonical_divergence() - .expect("divergence") - .is_none() - ); - let accepted = query_latest_safe_accepted_batch(&storage.conn) - .expect("accepted frontier") - .expect("resumed acceptance"); - assert_eq!((accepted.safe_input_index, accepted.nonce), (2, 1)); - assert_eq!( + .write(|tx| { + super::super::history::initialize_history_in( + tx, + ExecutedInputCount::new(41), + 30, + )?; + super::super::mutations::set_batch_tree_anchor_in(tx, 1)?; + super::super::ingress::open_recovery_tip_in_tx(tx, 30) + }) + .expect("install recovered baseline"); + let mut head = storage.open_state().expect("read root").expect("root"); storage - .conn - .query_row("SELECT COUNT(*) FROM safe_accepted_batches", [], |row| row - .get::<_, i64>( - 0 - )) - .expect("accepted count"), - 1 - ); + .close_frame_and_batch(&mut head, 30) + .expect("close resumed batch"); + let payload = local_batch_payload(&mut storage, 1); + storage + .append_safe_inputs( + 31, + &[StoredSafeInput { + sender: submitter, + payload, + block_number: 31, + }], + submitter, + &timing, + ) + .expect("accept post-baseline batch"); + assert!( + storage + .canonical_divergence() + .expect("divergence") + .is_none(), + "reinterpreted future-nonce input at block {old_future_block}" + ); + let accepted = query_latest_safe_accepted_batch(&storage.conn) + .expect("accepted frontier") + .expect("resumed acceptance"); + assert_eq!((accepted.safe_input_index, accepted.nonce), (2, 1)); + assert_eq!( + storage + .conn + .query_row("SELECT COUNT(*) FROM safe_accepted_batches", [], |row| row + .get::<_, i64>( + 0 + )) + .expect("accepted count"), + 1 + ); + } } fn insert_safe_input_zero(storage: &Storage) { From 8b6da04374c7a4aef61fba8e7963e71d3cd55122 Mon Sep 17 00:00:00 2001 From: gcdepaula Date: Sat, 19 Sep 2026 17:40:36 -0300 Subject: [PATCH 27/29] fix: preserve snapshots across filesystem mount aliases --- docs/snapshots/lifecycle.md | 17 ++-- sequencer/src/commands/run/startup_hygiene.rs | 77 +++++++++++++++++-- 2 files changed, 79 insertions(+), 15 deletions(-) diff --git a/docs/snapshots/lifecycle.md b/docs/snapshots/lifecycle.md index c16902c8..1cf78d96 100644 --- a/docs/snapshots/lifecycle.md +++ b/docs/snapshots/lifecycle.md @@ -135,10 +135,13 @@ of all checkpoint resources. Filesystem deletion failure leaves a harmless orphan for the startup sweep. The reverse ordering would leave a durable row pointing at missing state and is forbidden. Startup clears leases left by the dead process, checks the rollback checkpoint's `info.toml` and format version, collects obsolete -rows, and sweeps orphan directories before workers start. The sweep resolves -every retained artifact path before deleting orphans and compares resolved paths -so alternate spellings and symlinks preserve the same artifact. Application -restoration runs afterward in the launched inclusion lane, before processing new -user ops; the metadata check does not validate the application bytes. Missing or -corrupt referenced artifacts fail loud when read or restored; operational -filesystem errors retain their normal error classification. +rows, and sweeps orphan directories before workers start. On supported Linux and +macOS hosts, the sweep reads every retained artifact's filesystem object identity +(device and inode) before deleting any orphans. Comparing those identities +preserves alternate spellings, symlinks, and mount aliases of the same artifact. +Retained identity lookup failures stop the sweep before deletion: missing +references or structural I/O errors are terminal; operational I/O errors retain +their normal classification. Application restoration runs afterward in the launched inclusion +lane, before processing new user ops; the metadata check does not validate the +application bytes. Missing or corrupt referenced artifacts also fail loud when +read or restored. diff --git a/sequencer/src/commands/run/startup_hygiene.rs b/sequencer/src/commands/run/startup_hygiene.rs index b01136c2..bfbf7366 100644 --- a/sequencer/src/commands/run/startup_hygiene.rs +++ b/sequencer/src/commands/run/startup_hygiene.rs @@ -4,6 +4,8 @@ //! Startup clears stale leases, checks rollback checkpoint metadata, then collects //! obsolete snapshots and orphan directories before workers are admitted. +use std::os::unix::fs::MetadataExt; + use crate::commands::error::CommandError; use crate::ingress::inclusion_lane::dump_info::{self, delete_dump_dir}; @@ -61,7 +63,7 @@ fn snapshot_gc_at_startup(storage: &mut crate::storage::Storage) -> Result, _>>()?; @@ -85,8 +85,8 @@ fn sweep_orphan_dumps( for entry in std::fs::read_dir(dumps_dir)? { let entry = entry?; let path = entry.path(); - let retained = match std::fs::canonicalize(&path) { - Ok(resolved) => known.contains(&resolved), + let retained = match dump_identity(&path) { + Ok(identity) => known.contains(&identity), // GC or an earlier orphan deletion can leave an unregistered // dangling symlink. Retained references already resolved above. Err(err) if err.kind() == std::io::ErrorKind::NotFound => false, @@ -109,6 +109,12 @@ fn sweep_orphan_dumps( Ok(removed) } +fn dump_identity(path: &std::path::Path) -> std::io::Result<(u64, u64)> { + // Canonical paths can still differ across mount aliases and macOS firmlinks. + let metadata = std::fs::metadata(path)?; + Ok((metadata.dev(), metadata.ino())) +} + #[cfg(test)] mod tests { use super::*; @@ -294,6 +300,61 @@ mod tests { } } + #[cfg(target_os = "macos")] + #[test] + fn sweep_preserves_mixed_firmlink_and_literal_references() { + let db = temp_db("sweep-firmlink-alias"); + let mut storage = Storage::open(&db.path).unwrap(); + let root = tempfile::tempdir_in("/private/tmp").unwrap(); + let dumps = root.path().join("dumps"); + std::fs::create_dir(&dumps).unwrap(); + let alias = + std::path::Path::new("/System/Volumes/Data").join(dumps.strip_prefix("/").unwrap()); + let tracked = dumps.join("tracked"); + create_structured_dump(&tracked); + storage + .insert_baseline_snapshot(&tracked, crate::storage::ExecutedInputCount::ZERO) + .unwrap(); + + // This alias survives realpath resolution, unlike an ordinary symlink. + let listed = alias.join("tracked"); + assert_ne!( + tracked.canonicalize().unwrap(), + listed.canonicalize().unwrap() + ); + let stored_metadata = std::fs::metadata(&tracked).unwrap(); + let listed_metadata = std::fs::metadata(&listed).unwrap(); + assert_eq!( + (stored_metadata.dev(), stored_metadata.ino()), + (listed_metadata.dev(), listed_metadata.ino()) + ); + + // The sets overlap literally, so a global disjoint-set guard is insufficient. + let literal_match = alias.join("literal-match"); + create_structured_dump(&literal_match); + storage + .write(|tx| { + tx.execute( + "INSERT INTO dumps(prefix) VALUES (?1)", + [literal_match.to_str().unwrap()], + ) + }) + .unwrap(); + let orphan = dumps.join("orphan"); + create_structured_dump(&orphan); + + let removed = sweep_orphan_dumps(&mut storage, &alias).unwrap(); + + assert!( + tracked.join("info.toml").is_file(), + "the referenced artifact must survive its mount alias" + ); + assert!(literal_match.join("info.toml").is_file()); + assert!(!orphan.exists()); + assert_eq!(removed, 1); + assert_eq!(storage.list_dump_rows().unwrap().len(), 2); + } + #[test] fn sweep_resolves_every_reference_before_deleting_any_artifact() { let db = temp_db("sweep-unresolved-reference"); From 93e6a49ad19c35fe78f4c46d639eb8fd556c58ba Mon Sep 17 00:00:00 2001 From: gcdepaula Date: Sat, 19 Sep 2026 17:40:36 -0300 Subject: [PATCH 28/29] ci: preserve logs from failed rollups E2E runs --- .github/workflows/ci.yml | 9 +++++++++ docs/review/register.md | 12 ++++++++++++ 2 files changed, 21 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f319dbea..7ae4ff6b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -138,6 +138,15 @@ jobs: - name: Watchdog Lua CM e2e run: just test-watchdog-e2e + - name: Upload E2E failure logs + if: ${{ failure() }} + uses: actions/upload-artifact@v6 + with: + name: rollups-e2e-logs-${{ github.run_attempt }} + path: tests/e2e/results/*.log + retention-days: 7 + if-no-files-found: ignore + watchdog-docker: name: Watchdog Docker image smoke runs-on: ubuntu-latest diff --git a/docs/review/register.md b/docs/review/register.md index d9cf0b42..49ce13d3 100644 --- a/docs/review/register.md +++ b/docs/review/register.md @@ -60,6 +60,18 @@ exposure in an actual deployment was established by this review. descriptor/process ownership before changing the assertion or lock behavior. Reproduced during the 2026-09-18 stack closeout; isolated and full serial runs passed. See the [current validation record](2026-09-18-stack-review-validation.md). +- **Intermittent C-host recovery WebSocket reset.** At `dc4dd78` on + 2026-09-19, `c_host_recovery_after_stale_batches_test` failed an expected + message receive with `Connection reset without closing handshake` in the + [push run](https://github.com/cartesi/sequencer/actions/runs/35456024614/job/105931662252). + The [PR run](https://github.com/cartesi/sequencer/actions/runs/35456514750/job/105934071941) + passed all 49 scenarios on the same source tree. The failed run retained no + child-process log artifact, so the reset's cause is unclassified. CI now + uploads `tests/e2e/results/*.log` on failure. On recurrence, use those logs + to identify the server's last events and the failing receive before changing + timeouts, teardown, or retry behavior. Evidence: the + [recovery scenario](../../tests/e2e/src/test_cases.rs) and + [WS receive helper](../../tests/harness/src/ws.rs). - **Transient SQLite contention stops the submitter.** Read handles use a 50 ms busy timeout; a storage/open failure escapes the submitter loop. BUSY/LOCKED are nonterminal but project to unclassified exit 1, causing From 209a3b167999fd47c42e6cfba8ce53398ad6c0ed Mon Sep 17 00:00:00 2001 From: gcdepaula Date: Sat, 19 Sep 2026 17:41:30 -0300 Subject: [PATCH 29/29] docs: clarify checkpoint eligibility and refresh review evidence --- README.md | 11 +++--- docs/recovery/cockroach.md | 8 +++- .../2026-09-19-review-followup-validation.md | 37 ++++++++++++++----- sequencer/src/commands/error.rs | 6 ++- 4 files changed, 43 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index ee68b2f6..7d077933 100644 --- a/README.md +++ b/README.md @@ -167,6 +167,12 @@ Most queue sizes, polling intervals, and safety limits are now internal runtime ## API +JSON `sender` fields in successful `POST /tx` responses and WebSocket messages +use EIP-55 checksum casing. Address fields in `/history` and `sender` fields in +`/historical-l1-inputs` use lowercase hex. Clients must compare decoded 20-byte +addresses and use one normalized encoding for account or projection keys across +these routes. + ### `POST /tx` Request shape: @@ -263,11 +269,6 @@ L1 and then join the application feed. The [projection replay contract](docs/protocol/projection-replay.md) describes bootstrap, client checkpoints, pending directs, and terminal drain. -WS `sender` strings use EIP-55 checksum casing; address fields in `/history` -and `sender` strings in `/historical-l1-inputs` use lowercase hex. Clients must -compare decoded 20-byte addresses and use one normalized encoding for projection -keys across these feeds. - `GET /history` returns one coherent view of the deployment, current application history, immutable era baseline, and latest accepted checkpoint. Optional `era_id=` requires the selected era; a mismatch returns `409 ERA_CHANGED`. diff --git a/docs/recovery/cockroach.md b/docs/recovery/cockroach.md index 72bbab51..34be82c6 100644 --- a/docs/recovery/cockroach.md +++ b/docs/recovery/cockroach.md @@ -208,8 +208,12 @@ If it fails, refuse the export and select an eligible earlier checkpoint or genesis. Honest live sequencing establishes this condition: a frame's safe block precedes -its L1 inclusion, so all directs it covers have already arrived. The canonical -scheduler also accepts equality, however. After faulty sequencing, a batch at +its L1 inclusion, so all directs it covers have already arrived. The overdue-direct +backstop preserves it too: by the time a block's directs are overdue, that whole +block has been observed. The FIFO drain executes all equally aged directs from +that block and all older ones, so advancing `A` leaves none of them pending. + +The canonical scheduler also accepts equality. After faulty sequencing, a batch at block 10 can execute at clock 10 before another direct in that block arrives. An empty batch at block 11 advances the nonce without draining that direct. The truthful checkpoint has `A=10 < B=11`, yet seeding `(A,B]` would omit it. diff --git a/docs/review/2026-09-19-review-followup-validation.md b/docs/review/2026-09-19-review-followup-validation.md index 68380426..8b1a0063 100644 --- a/docs/review/2026-09-19-review-followup-validation.md +++ b/docs/review/2026-09-19-review-followup-validation.md @@ -2,16 +2,23 @@ Evidence for the ongoing stack review and landing, covering changes above `f504c2e88e2b432718493e29e3c56f74fce3ad00` on `codex/stack-review-fixes`. +Local validation below includes the filesystem-identity fix at +`8b6da04374c7a4aef61fba8e7963e71d3cd55122`, CI diagnostics at +`93e6a49ad19c35fe78f4c46d639eb8fd556c58ba`, and the accompanying documentation +clarifications. Those clarifications change no executable behavior. Retire this record after landing when no ongoing review decision uses it. ## Changes and discriminating checks -- Startup compares resolved artifact paths after resolving every retained - reference. Regressions preserve relative/absolute, leading-dot, and symlink - aliases, including mixed literal/aliased references, while removing genuine - orphans. A missing reference stops the sweep before any deletion. Dangling - orphan links are removed without following their targets. All ten startup - hygiene tests pass. +- Startup compares filesystem device/inode identities after inspecting every + retained reference. Regressions preserve relative/absolute, leading-dot, + symlink, and macOS firmlink aliases, including mixed literal/aliased references, + while removing genuine orphans. The firmlink test fails against the old + canonical-path comparison because it deletes the registered artifact, and + passes with identity matching. A missing reference stops the sweep before any + deletion. Dangling orphan links are removed without following their targets. + All eleven startup hygiene tests pass. The firmlink regression runs on macOS; + no Linux bind-mount scenario was run locally. - A scheduler/fold regression establishes that `A < B` can coexist with a pending direct at `A` after a same-block frame and a later empty batch. It verifies recovery from an eligible earlier checkpoint preserves every direct, @@ -26,6 +33,9 @@ Retire this record after landing when no ongoing review decision uses it. WS admission statuses, health-probe semantics, and internal deployment boundary. Writer ownership and stale source references were corrected. The submitter's own-sender decode error remains visible under self-trust. +- CI retains harness logs from failed rollups E2E jobs for seven days. This adds + evidence for the unclassified C-host recovery WebSocket reset in the + [register](register.md), without changing test or recovery behavior. Separate reviewers examined snapshot cleanup and checkpoint eligibility. Review caught the dangling-orphan-link case before final validation; its regression is @@ -37,11 +47,12 @@ macOS arm64, parent Nix/direnv environment, Rust and Cargo 1.95.0: | Check | Result | |---|---| -| `cargo test --locked --workspace --exclude canonical-test -- --test-threads=1` | 759 passed; zero failed; one existing ignored harness doc test | +| `cargo test --locked --workspace --exclude canonical-test -- --test-threads=1` | 760 passed; zero failed; one existing ignored harness doc test | | `cargo check --locked --workspace --all-targets` | Passed | | `cargo clippy --locked --workspace --all-targets --all-features -- -D warnings` | Passed | | `cargo fmt --all -- --check` and `git diff --check` | Passed | -| Relative Markdown links in the changed contract/API documents | 111 targets/anchors checked | +| Relative Markdown links and code fences across repository Markdown files | 409 targets/anchors checked | +| CI workflow YAML parsing | Passed; artifact upload itself requires a failed GitHub job | The full host suite includes the new regressions. Guest execution, standalone rollups E2Es, watchdog Lua tests, and TLA+ model runs were not repeated for this @@ -61,5 +72,11 @@ tracks that work. Canonical acceptance semantics and artifact formats are unchan Archive concurrency limits remain deferred pending deployment workload needs. The existing process-lock concurrency investigation remains open; this host run -used the serial suite. The lower-stack C-host ancestry reconciliation remains -landing work. No remote branch, PR discussion, or merge was changed. +used the serial suite. The C-host WebSocket reset is a separate unresolved +investigation; one successful run cannot classify its cause. + +The lower-stack ancestry is reconciled: PR #38 is at +`7f3229f2e42585e055f2fabb8e280c4d42dd5a81`, and GitHub reports PR #42 mergeable +at `35697691d7a5ba5a2c868f51b3a45c3dd5b6ee44` (checked 2026-09-19). +Consult CI for the pushed revision before landing; local host checks do not +replace guest, watchdog, or standalone rollups E2E execution. diff --git a/sequencer/src/commands/error.rs b/sequencer/src/commands/error.rs index 5ed45a76..28204902 100644 --- a/sequencer/src/commands/error.rs +++ b/sequencer/src/commands/error.rs @@ -411,8 +411,10 @@ pub enum SetupRecoveryError { /// recovery export (`info.toml` + `checkpoint.toml` + `state/`), not a watchdog CM checkpoint. #[error("failed to load checkpoint dump at {path}: {message}")] CheckpointLoad { path: String, message: String }, - /// Outside the known empty genesis checkpoint, A must precede B so the - /// recovery seed includes all potentially pending directs in block B. + /// Outside empty genesis, A must precede B: equality can exclude pending + /// directs in B from the seed range. A < B alone does not certify queue + /// eligibility; the exporter must also verify no pending direct is at or + /// below A (docs/recovery/cockroach.md). #[error( "checkpoint last-executed safe block {executed_safe_block} (A) must precede \ checkpoint block {checkpoint_block} (B), except for empty genesis; \