diff --git a/README.md b/README.md index eb2c85b1..665d7e36 100644 --- a/README.md +++ b/README.md @@ -235,6 +235,7 @@ def test_cached_function(): - Connection pooling with thread affinity (+28% throughput) - Distributed locking prevents cache stampedes - Pluggable backend abstraction (Redis, CachekitIO, File, Memcached, custom) +- Untrusted-decode bounds: nesting depth and header-declared allocation are capped on every cache read (a forged entry is a bounded cache miss), verified against the protocol's shared [`decode-bounds.json`](https://github.com/cachekit-io/protocol/blob/2d56cce231e193141f09df9316f9afac17a1538e/test-vectors/decode-bounds.json) vectors > [!NOTE] > All reliability features are **enabled by default** with `@cache.production`. Use `@cache.minimal` to disable them for maximum throughput. diff --git a/rust/src/lib.rs b/rust/src/lib.rs index 8afa7f23..fca8d5b7 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -1,11 +1,15 @@ //! `PyO3` bindings for `cachekit-core` //! //! This crate provides thin Python wrappers around the cachekit-core library. -//! All business logic lives in cachekit-core; this crate only handles Python FFI. +//! Business logic lives in cachekit-core, with one SDK-owned exception: the untrusted +//! msgpack decode bound in `msgpack_bounds` (LAB-2503), pending a core-shared walk. // Re-export core types for use in Python bindings pub use cachekit_core::{ByteStorage, OperationMetrics, StorageEnvelope}; +/// Untrusted msgpack structural bound — pure Rust, not gated on `python` +pub mod msgpack_bounds; + #[cfg(feature = "encryption")] pub use cachekit_core::{ derive_domain_key, @@ -40,6 +44,13 @@ fn _rust_serializer(_py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(python_bindings::checksum_py, m)?)?; m.add_function(wrap_pyfunction!(python_bindings::verify_checksum_py, m)?)?; + // Untrusted-decode structural bound (LAB-2503) — zero-copy header walk that + // serializers/base.py::unpackb_bounded runs before every msgpack.unpackb + m.add_function(wrap_pyfunction!( + python_bindings::check_msgpack_structure_py, + m + )?)?; + // Add encryption functionality if feature is enabled #[cfg(feature = "encryption")] { diff --git a/rust/src/msgpack_bounds.rs b/rust/src/msgpack_bounds.rs new file mode 100644 index 00000000..ef87a77d --- /dev/null +++ b/rust/src/msgpack_bounds.rs @@ -0,0 +1,87 @@ +//! Structural bound for untrusted MessagePack (LAB-2503; protocol spec/interop-mode.md → +//! Decode bounds). The one algorithm this crate owns rather than delegates to cachekit-core; +//! a core-shared walk usable from py/rs/wasm is the follow-up. Mirrors the opcode table of +//! cachekit-rs `check_structure` so the two SDKs reject the same documents. + +/// Header-only walk over one `MessagePack` document: str/bin/ext payloads are skipped by +/// offset, never read, and nothing is allocated beyond one `u64` per open collection. +/// +/// Trailing bytes after the root element are left to the decoder (`ExtraData`). +/// +/// # Errors +/// +/// Names the violated bound, before any decoder pre-allocates a container, for: +/// - nesting deeper than `max_depth`; +/// - a header declaring more payload bytes than the input holds; +/// - more pending elements (across every open collection) than remaining bytes can back — +/// every element costs >= 1 byte, so a decoder's total container pre-allocation is then +/// bounded by the input length instead of by `depth × declared_len`; +/// - the reserved marker 0xc1 and input that ends mid-document. +pub fn check_msgpack_structure(bytes: &[u8], max_depth: usize) -> Result<(), String> { + fn be(bytes: &[u8], pos: usize, width: usize) -> Result { + let end = pos + .checked_add(width) + .filter(|e| *e <= bytes.len()) + .ok_or_else(|| "ends inside a length prefix".to_owned())?; + Ok(bytes[pos..end] + .iter() + .fold(0u64, |acc, b| (acc << 8) | u64::from(*b))) + } + + let mut pos = 0usize; + let mut pending: u64 = 1; // elements owed across all open collections (the root is one) + let mut open: Vec = Vec::new(); // elements still owed per open collection = depth + while pending > 0 { + while open.last() == Some(&0) { + open.pop(); + } + let marker = *bytes + .get(pos) + .ok_or_else(|| "ends before the document is complete".to_owned())?; + pos += 1; + pending -= 1; + if let Some(innermost) = open.last_mut() { + *innermost -= 1; + } + // (length-prefix bytes, payload bytes after the prefix, child elements) + let (prefix, payload, children): (usize, u64, u64) = match marker { + 0x00..=0x7f | 0xc0 | 0xc2 | 0xc3 | 0xe0..=0xff => (0, 0, 0), + 0x80..=0x8f => (0, 0, 2 * u64::from(marker & 0x0f)), + 0x90..=0x9f => (0, 0, u64::from(marker & 0x0f)), + 0xa0..=0xbf => (0, u64::from(marker & 0x1f), 0), + 0xc1 => return Err("contains the reserved marker 0xc1".to_owned()), + 0xc4 | 0xd9 => (1, be(bytes, pos, 1)?, 0), + 0xc5 | 0xda => (2, be(bytes, pos, 2)?, 0), + 0xc6 | 0xdb => (4, be(bytes, pos, 4)?, 0), + 0xc7 => (1, be(bytes, pos, 1)? + 1, 0), // ext: length prefix, then type byte + data + 0xc8 => (2, be(bytes, pos, 2)? + 1, 0), + 0xc9 => (4, be(bytes, pos, 4)? + 1, 0), + 0xca..=0xd3 => (0, 1u64 << (marker & 0x03), 0), // f32/f64/u8..u64/i8..i64: 4,8,1,2,4,8,1,2,4,8 + 0xd4..=0xd8 => (0, 1 + (1u64 << (marker - 0xd4)), 0), // fixext: type byte + 1/2/4/8/16 + 0xdc => (2, 0, be(bytes, pos, 2)?), + 0xdd => (4, 0, be(bytes, pos, 4)?), + 0xde => (2, 0, 2 * be(bytes, pos, 2)?), + 0xdf => (4, 0, 2 * be(bytes, pos, 4)?), + }; + pos += prefix; + let remaining = (bytes.len() - pos) as u64; + if payload > remaining { + return Err("declares more bytes than the input holds".to_owned()); + } + // <= remaining, so this cannot fail; `try_from` rather than `as usize` satisfies + // clippy::cast_possible_truncation, line-for-line with cachekit-rs `check_structure`. + pos += usize::try_from(payload) + .map_err(|_| "declares more bytes than the input holds".to_owned())?; + if children > 0 { + if open.len() >= max_depth { + return Err(format!("nests deeper than {max_depth} levels")); + } + open.push(children); + } + pending += children; + if pending > remaining - payload { + return Err("declares more elements than the input can back".to_owned()); + } + } + Ok(()) +} diff --git a/rust/src/python_bindings.rs b/rust/src/python_bindings.rs index d89cf553..d15c96e7 100644 --- a/rust/src/python_bindings.rs +++ b/rust/src/python_bindings.rs @@ -1,8 +1,10 @@ //! Python bindings for cachekit-core //! -//! This module provides thin PyO3 wrappers around cachekit-core functionality. -//! All business logic is delegated to cachekit-core. +//! This module provides thin PyO3 wrappers around cachekit-core functionality, plus the +//! buffer-borrow helper they share. Business logic lives in cachekit-core, except the +//! SDK-owned msgpack decode bound in `crate::msgpack_bounds`. +use crate::msgpack_bounds::check_msgpack_structure; use cachekit_core::ByteStorage; use pyo3::buffer::PyBuffer; use pyo3::exceptions::PyValueError; @@ -44,6 +46,67 @@ fn borrowable_offset(buf: &PyBuffer, base: &Bound<'_, PyBytes>) -> Option { + /// `(base, offset, len)`: a window onto an immutable `bytes` object kept alive by the + /// Bound — the whole object, or the read-only C-contiguous `memoryview` of it that + /// `SerializationWrapper.unwrap` produces, proven by `borrowable_offset`. Zero-copy. + Borrowed(Bound<'py, PyBytes>, usize, usize), + /// Mutable, non-`bytes`-backed, strided, or empty exporter: the only safe answer is a copy. + Owned(Vec), +} + +impl BytesView<'_> { + fn as_slice(&self) -> &[u8] { + match self { + BytesView::Borrowed(base, off, len) => &base.as_bytes()[*off..*off + *len], + BytesView::Owned(v) => v, + } + } +} + +/// Borrow `obj`'s bytes zero-copy when the BACKING STORAGE is provably immutable, else copy. +/// +/// `readonly()` describes the view, not the exporter (`memoryview(bytearray).toreadonly()` +/// passes it while another thread can still mutate the bytearray), and a PEP 688 +/// `__buffer__` exporter can name a decoy `bytes` in `.obj` — so the gate is the containment +/// proof in `borrowable_offset`, whose payoff is that the borrow is an ORDINARY SLICE of that +/// `bytes`: bounds-checked by Rust, no `unsafe`, nothing for a stale comment to misstate. +fn bytes_view<'py>(py: Python<'py>, obj: &Bound<'py, PyAny>) -> PyResult> { + if let Ok(b) = obj.cast::() { + return Ok(BytesView::Borrowed(b.clone(), 0, b.len()?)); + } + let buf = PyBuffer::::get(obj)?; + let base = obj + .getattr("obj") + .ok() + .and_then(|base| base.cast_into::().ok()); + if let Some(base) = base { + if let Some(off) = borrowable_offset(&buf, &base) { + return Ok(BytesView::Borrowed(base, off, buf.item_count())); + } + } + Ok(BytesView::Owned(buf.to_vec(py)?)) +} + +/// Reject a MessagePack document whose headers would make decoding it allocate out of +/// proportion to its size — see `check_msgpack_structure`. Zero-copy for `bytes` and for +/// read-only `memoryview`s of `bytes`; raises ValueError naming the violated bound. +#[pyfunction] +#[pyo3(name = "check_msgpack_structure")] +pub fn check_msgpack_structure_py( + py: Python<'_>, + data: &Bound<'_, PyAny>, + max_depth: usize, +) -> PyResult<()> { + let view = bytes_view(py, data)?; + check_msgpack_structure(view.as_slice(), max_depth).map_err(|what| { + PyValueError::new_err(format!("Unpack failed: MessagePack document {what}")) + }) +} + #[pymethods] impl PyByteStorage { #[new] @@ -86,41 +149,10 @@ impl PyByteStorage { py: Python, envelope_bytes: &Bound<'_, PyAny>, ) -> PyResult<(Vec, String)> { - let owned: Vec; - let buf: PyBuffer; - let base_bytes: Option>; - let data: &[u8] = if let Ok(b) = envelope_bytes.cast::() { - // `bytes` is immutable and kept alive by the Bound for the whole call: - // a zero-copy borrow with no data-race exposure. - b.as_bytes() - } else { - buf = PyBuffer::get(envelope_bytes)?; - // Borrowing across the GIL release below is only sound when the BACKING - // STORAGE is immutable — readonly() describes the view, not the exporter - // (memoryview(bytearray).toreadonly() passes it while another thread can - // still mutate the bytearray). Attribute trust is not enough either: a - // PEP 688 __buffer__ exporter can name a decoy `bytes` in `.obj`. So the - // gate is a containment proof (borrowable_offset), and its payoff is that - // the borrow becomes expressible as an ORDINARY SLICE of that `bytes` — - // bounds-checked by Rust, no `unsafe`, nothing for a stale comment to - // misstate. Anything unproven falls back to a copy. - base_bytes = envelope_bytes - .getattr("obj") - .ok() - .and_then(|base| base.cast_into::().ok()); - let borrowed = base_bytes.as_ref().and_then(|base| { - borrowable_offset(&buf, base) - .map(|off| &base.as_bytes()[off..off + buf.item_count()]) - }); - match borrowed { - Some(slice) => slice, - None => { - // Mutable, non-bytes-backed, non-contiguous, or empty exporter. - owned = buf.to_vec(py)?; - &owned - } - } - }; + // Borrowing across the GIL release below is only sound when the backing storage + // is immutable — bytes_view proves that or copies (see its doc). + let view = bytes_view(py, envelope_bytes)?; + let data = view.as_slice(); // Detach from the GIL for decompression + checksum (see store()). py.detach(|| self.inner.retrieve(data)) .map_err(|e| PyValueError::new_err(format!("Retrieval failed: {}", e))) diff --git a/src/cachekit/interop.py b/src/cachekit/interop.py index 6be28ee9..d15a9bc4 100644 --- a/src/cachekit/interop.py +++ b/src/cachekit/interop.py @@ -36,9 +36,7 @@ from typing import Any from uuid import UUID -import msgpack - -from .serializers.base import SerializationError +from .serializers.base import SerializationError, unpackb_bounded # Full-string match REQUIRED: re.match with a $ anchor still accepts a # trailing newline. Pinned by the reject_trailing_newline error vector. @@ -440,7 +438,7 @@ def decode_interop_value(data: bytes | bytearray | memoryview) -> Any: "check that every writer for this key uses @cache(interop=...)." ) try: - return msgpack.unpackb(raw, raw=False, strict_map_key=True, object_hook=_revive_sentinels) + return unpackb_bounded(raw, raw=False, strict_map_key=True, object_hook=_revive_sentinels) except Exception as e: raise InteropDecodeError(f"stored value is not a single well-formed MessagePack document: {e}") from e diff --git a/src/cachekit/serializers/auto_serializer.py b/src/cachekit/serializers/auto_serializer.py index 1dcd25ce..f75f7bb0 100644 --- a/src/cachekit/serializers/auto_serializer.py +++ b/src/cachekit/serializers/auto_serializer.py @@ -60,7 +60,7 @@ from cachekit._rust_serializer import ByteStorage -from .base import SerializationError, SerializationFormat, SerializationMetadata +from .base import PAYLOAD_DECODE_ERRORS, SerializationError, SerializationFormat, SerializationMetadata, unpackb_bounded logger = logging.getLogger(__name__) @@ -157,6 +157,51 @@ def _is_plain_numpy_numeric(dtype: Any) -> bool: return HAS_PANDAS and not pd.api.types.is_extension_array_dtype(dtype) and dtype.kind in ("i", "u", "f") +def _dtype_from_untrusted(spec: Any, *, numeric_only: bool = False) -> np.dtype: + """``np.dtype(spec)`` for a dtype the cache entry itself supplies, refusing what the writer never emits. + + A forged ``M8[0ns]`` (zero datetime unit multiplier) passes ``np.frombuffer`` and then kills + the process with SIGFPE inside pandas — a signal no ``except`` can catch — so it is refused + before any array is built. Columnar (DataFrame/Series) entries only ever carry dtypes that + pass ``_is_plain_numpy_numeric``, the write-side predicate, so ``numeric_only`` mirrors it. + """ + dtype = np.dtype(spec) + if numeric_only and not _is_plain_numpy_numeric(dtype): + raise SerializationError(f"Forged columnar dtype {dtype}: the writer only emits plain NumPy numeric columns") + if dtype.kind in "Mm" and np.datetime_data(dtype)[1] == 0: + raise SerializationError(f"Forged dtype {dtype}: a zero datetime unit multiplier crashes pandas") + return dtype + + +def _expect(value: Any, kind: type, what: str) -> Any: + """Refuse a columnar field whose type the writer never emits. + + The ``__ndarray__`` object hook can substitute an attacker-typed ndarray for any field of a + forged DataFrame/Series document; pandas then asserts (``AssertionError``) or indexing raises + ``IndexError`` — both outside ``PAYLOAD_DECODE_ERRORS``. The writer emits ``list`` for + ``columns`` / ``index`` / object data and ``dict`` for the document and each column. + """ + if not isinstance(value, kind): + raise SerializationError(f"Forged columnar payload: {what} is {type(value).__name__}, expected {kind.__name__}") + return value + + +def _column_values(info: dict[str, Any], what: str) -> Any: + """Rebuild one column's values from the ``{type, data[, dtype]}`` the writer emits (``dtype`` only for ``"numeric"``). + + ``type`` is an allow-list, not a numeric/else switch: an unknown marker must not be read as object data. + """ + marker = info["type"] + if marker == "numeric": + # .copy() → writable values that do not alias the source buffer (#157). + return np.frombuffer(info["data"], dtype=_dtype_from_untrusted(info["dtype"], numeric_only=True)).copy() + if marker == "object": + return _expect(info["data"], list, f"{what} data") + # Attacker-chosen: echo a str capped at 40 chars; never repr() a structure (RecursionError on 3.10/3.11 at depth ~1000). + shown = marker if isinstance(marker, str) else type(marker).__name__ + raise SerializationError(f"Forged columnar payload: {what} type is {shown!r:.40}, expected 'numeric' or 'object'") + + def _na_safe_object_list(series: Any) -> list: """``series.tolist()`` with scalar pandas NA sentinels (pd.NA/NaT/NaN) mapped to None. @@ -297,7 +342,7 @@ def _auto_object_hook(obj: Any) -> Any: if "data" not in obj or "shape" not in obj or "dtype" not in obj: raise SerializationError("Invalid ndarray format: missing required fields in cached data") # .copy(): writable result that does not alias the source buffer (the L1-cached bytes on a hit) — #157. - return np.frombuffer(obj["data"], dtype=obj["dtype"]).reshape(obj["shape"]).copy() + return np.frombuffer(obj["data"], dtype=_dtype_from_untrusted(obj["dtype"])).reshape(obj["shape"]).copy() return obj @@ -550,11 +595,9 @@ def deserialize(self, data: bytes | memoryview, metadata: Optional[Serialization original_data, _ = self._byte_storage.retrieve(data) except (ValueError, SerializationError) as e: raise SerializationError(f"DataFrame integrity check failed (corrupted cache entry): {e}") from e - unpacked_data = msgpack.unpackb(original_data, **self._msgpack_unpack_opts) - return self._deserialize_dataframe(unpacked_data) + return self._decode_columnar(original_data, detected_format) # Integrity off: data is direct msgpack (no envelope) - unpacked_data = msgpack.unpackb(data, **self._msgpack_unpack_opts) - return self._deserialize_dataframe(unpacked_data) + return self._decode_columnar(data, detected_format) elif detected_format == "series": if self.enable_integrity_checking and len(data) > 4: # Same fail-closed contract as the DataFrame branch above (#156). @@ -562,43 +605,47 @@ def deserialize(self, data: bytes | memoryview, metadata: Optional[Serialization original_data, _ = self._byte_storage.retrieve(data) except (ValueError, SerializationError) as e: raise SerializationError(f"Series integrity check failed (corrupted cache entry): {e}") from e - unpacked_data = msgpack.unpackb(original_data, **self._msgpack_unpack_opts) - return self._deserialize_series(unpacked_data) + return self._decode_columnar(original_data, detected_format) # Integrity off: data is direct msgpack (no envelope) - unpacked_data = msgpack.unpackb(data, **self._msgpack_unpack_opts) - return self._deserialize_series(unpacked_data) + return self._decode_columnar(data, detected_format) # For Rust-envelope formats, use the Rust layer + envelope_error: Exception | None = None if self.enable_integrity_checking: try: # Use Rust layer for decompression and validation original_data, format_id = self._byte_storage.retrieve(data) - - # Use metadata if available, otherwise fall back to format_id from envelope - if metadata and hasattr(metadata, "original_type"): - detected_format = metadata.original_type - else: - detected_format = format_id - - # Deserialize based on detected format - if detected_format == "numpy": - return self._deserialize_numpy(original_data) - elif detected_format == "dataframe": - # Unpack the msgpack data first, then pass to DataFrame deserializer - unpacked_data = msgpack.unpackb(original_data, **self._msgpack_unpack_opts) - return self._deserialize_dataframe(unpacked_data) - elif detected_format == "series": - # Unpack the msgpack data first, then pass to Series deserializer - unpacked_data = msgpack.unpackb(original_data, **self._msgpack_unpack_opts) - return self._deserialize_series(unpacked_data) - else: # msgpack - return msgpack.unpackb(original_data, **self._msgpack_unpack_opts) except SerializationError: # Re-raise SerializationError (corruption detection) without swallowing raise except Exception as e: - # If Rust envelope parsing fails for other reasons, try Python-only deserialization + # Not a ByteStorage envelope (e.g. written with integrity checking off): + # fall through to the Python-only paths below, keeping the reason for the + # final error (a checksum mismatch also lands here — retrieve raises a plain + # ValueError for both; distinguishing them is a Rust-extension follow-up). + envelope_error = e logger.debug(f"Rust envelope parsing failed, falling back to Python-only deserialization: {e}") + else: + # The envelope verified (checksum matched), so its payload is exactly what was + # stored; a payload that then fails to decode is corruption or a forged entry + # (LAB-2503 decode bomb) and MUST fail closed. Falling through here used to + # re-decode the ENVELOPE bytes as plain MessagePack and return its positional + # fields as the cached value — wrong data, silently. + # Use metadata if available, otherwise fall back to format_id from envelope + detected_format = metadata.original_type if metadata and hasattr(metadata, "original_type") else format_id + try: + if detected_format == "numpy": + return self._deserialize_numpy(original_data) + if detected_format in ("dataframe", "series"): + unpacked_data = unpackb_bounded(original_data, **self._msgpack_unpack_opts) + if detected_format == "dataframe": + return self._deserialize_dataframe(unpacked_data) + return self._deserialize_series(unpacked_data) + return unpackb_bounded(original_data, **self._msgpack_unpack_opts) + except PAYLOAD_DECODE_ERRORS as e: + raise SerializationError( + f"Cache entry payload failed to decode inside a verified envelope (format={detected_format!r}): {e}" + ) from e # Check for Arrow IPC format before msgpack fall-through # Arrow data may have xxHash3-64 checksum prefix (8 bytes) or be direct Arrow IPC @@ -621,13 +668,17 @@ def deserialize(self, data: bytes | memoryview, metadata: Optional[Serialization # Python-only path (no Rust compression) - direct msgpack deserialization try: - return msgpack.unpackb(data, **self._msgpack_unpack_opts) - except SerializationError: - # Re-raise SerializationError (corruption detection) without swallowing - raise - except Exception: - # If msgpack fails for other reasons, try NumPy-specific deserialization - return self._deserialize_numpy(data) + return unpackb_bounded(data, **self._msgpack_unpack_opts) + except PAYLOAD_DECODE_ERRORS as msgpack_error: + # NUMPY_RAW entries were routed structurally at the top, so nothing reaching here can be + # a NumPy payload (and a NumPy attempt would raise RuntimeError without the [data] + # extra). Report every reason for the miss: the msgpack one is the decode-bound + # rejection for a forged entry and must not vanish behind the envelope error. + raise SerializationError( + "Cache entry is not a decodable MessagePack payload" + f"{f' (envelope: {envelope_error})' if envelope_error else ''}" + f" (msgpack: {msgpack_error})" + ) from msgpack_error def _serialize_numpy(self, arr: np.ndarray) -> bytes: # type: ignore[name-defined] """Serialize a NumPy array into the ``NUMPY_RAW`` binary format. @@ -701,7 +752,7 @@ def _deserialize_numpy(self, data: bytes) -> np.ndarray: # Read dtype dtype_len = int.from_bytes(data[offset : offset + 2], byteorder="little") offset += 2 - dtype_str = data[offset : offset + dtype_len].decode("utf-8") + dtype_bytes = data[offset : offset + dtype_len] offset += dtype_len # Read shape @@ -710,6 +761,13 @@ def _deserialize_numpy(self, data: bytes) -> np.ndarray: shape_data = data[offset : offset + shape_len] offset += shape_len + # Slicing past the end silently shortens, and a partial 4-byte chunk would parse as a + # dimension (a forged 1-byte zero chunk = shape (0,) = an empty array instead of an + # error). Untrusted metadata must be exactly what its length prefix claims. + if len(dtype_bytes) != dtype_len or len(shape_data) != shape_len or shape_len % 4: + raise SerializationError("Invalid NumPy data format - truncated or misaligned dtype/shape metadata") + dtype_str = dtype_bytes.decode("utf-8") + # Reconstruct shape from packed integers shape = [] for i in range(0, len(shape_data), 4): @@ -721,9 +779,12 @@ def _deserialize_numpy(self, data: bytes) -> np.ndarray: # not alias the source bytes (the L1-cached buffer on a hit) — see #157. frombuffer alone # returns a read-only view aliasing the input. raw_bytes = data[offset:] - arr = np.frombuffer(raw_bytes, dtype=dtype_str).copy() + arr = np.frombuffer(raw_bytes, dtype=_dtype_from_untrusted(dtype_str)).copy() return arr.reshape(shape) - except (ValueError, IndexError, UnicodeDecodeError) as e: + except (ValueError, TypeError, IndexError, SyntaxError) as e: + # TypeError: np.frombuffer on a forged dtype string; SyntaxError: numpy's comma-string + # dtype parser runs ast.literal_eval on a forged shape prefix such as "(1,f8"; + # UnicodeDecodeError is a ValueError. raise SerializationError(f"Failed to deserialize NumPy array: {e}") from e def _serialize_dataframe(self, df: pd.DataFrame) -> bytes: @@ -767,6 +828,7 @@ def _deserialize_dataframe(self, data) -> pd.DataFrame: Raises: RuntimeError: If pandas not installed + SerializationError: forged document shape — see ``_expect`` / ``_column_values`` """ if not HAS_PANDAS: raise RuntimeError("Pandas not installed. Install with: pip install cachekit[data]") @@ -776,24 +838,18 @@ def _deserialize_dataframe(self, data) -> pd.DataFrame: serialized = data else: # Otherwise unpack msgpack - serialized = msgpack.unpackb(data, **self._msgpack_unpack_opts) + serialized = unpackb_bounded(data, **self._msgpack_unpack_opts) - # Reconstruct DataFrame column by column + serialized = _expect(serialized, dict, "document") columns_data = {} - for col, col_info in serialized["data"].items(): - if col_info["type"] == "numeric": - # Reconstruct from NumPy bytes; .copy() → writable, non-aliasing column (#157). - arr = np.frombuffer(col_info["data"], dtype=col_info["dtype"]).copy() - columns_data[col] = arr - else: - # Use object data directly - columns_data[col] = col_info["data"] - - df = pd.DataFrame(columns_data, columns=serialized["columns"]) + for col, col_info in _expect(serialized["data"], dict, "data").items(): + what = f"column {col!r:.40}" # col is attacker-chosen: cap the echo + columns_data[col] = _column_values(_expect(col_info, dict, what), what) + df = pd.DataFrame(columns_data, columns=_expect(serialized["columns"], list, "columns")) # Restore index if it was serialized if serialized["index"] is not None: - df.index = pd.Index(serialized["index"]) + df.index = pd.Index(_expect(serialized["index"], list, "index")) return df @@ -834,6 +890,7 @@ def _deserialize_series(self, data) -> pd.Series: Raises: RuntimeError: If pandas not installed + SerializationError: forged document shape — see ``_expect`` / ``_column_values`` """ if not HAS_PANDAS: raise RuntimeError("Pandas not installed. Install with: pip install cachekit[data]") @@ -843,22 +900,31 @@ def _deserialize_series(self, data) -> pd.Series: serialized = data else: # Otherwise unpack msgpack - serialized = msgpack.unpackb(data, **self._msgpack_unpack_opts) - - if serialized["type"] == "numeric": - # .copy() → writable Series values that do not alias the source buffer (#157). - values = np.frombuffer(serialized["data"], dtype=serialized["dtype"]).copy() - else: - values = serialized["data"] + serialized = unpackb_bounded(data, **self._msgpack_unpack_opts) - series = pd.Series(values, name=serialized["name"]) + serialized = _expect(serialized, dict, "document") + series = pd.Series(_column_values(serialized, "series"), name=serialized["name"]) # Restore index if it was serialized if serialized["index"] is not None: - series.index = pd.Index(serialized["index"]) + series.index = pd.Index(_expect(serialized["index"], list, "index")) return series + def _decode_columnar(self, payload: bytes | bytearray | memoryview, kind: str) -> pd.DataFrame | pd.Series: + """Decode a ``dataframe`` / ``series`` payload, failing closed as ``SerializationError``. + + The metadata routes in ``deserialize`` reach here outside the verified-envelope + normaliser, and the read handler treats only ``SerializationError`` as a read error + (evict + tamper hook) — a bare ``ValueError`` from the decode bound would be logged as + a backend fault and the poisoned entry kept (LAB-2503). + """ + build = self._deserialize_dataframe if kind == "dataframe" else self._deserialize_series + try: + return build(unpackb_bounded(payload, **self._msgpack_unpack_opts)) + except PAYLOAD_DECODE_ERRORS as e: + raise SerializationError(f"Cache entry payload failed to decode as {kind}: {e}") from e + def _serialize_msgpack(self, obj: Any) -> bytes: """Serialize general object with MessagePack.""" # Pre-process tuples into markers (msgpack natively flattens them to lists) @@ -908,10 +974,9 @@ def validate_data(self, data: bytes) -> bool: else: # Python-only mode validation try: - msgpack.unpackb(data, **self._msgpack_unpack_opts) + unpackb_bounded(data, **self._msgpack_unpack_opts) return True - except (msgpack.exceptions.UnpackException, ValueError, TypeError, AttributeError): - # AttributeError can occur when datetime_object_hook tries to restore invalid data + except PAYLOAD_DECODE_ERRORS: return False diff --git a/src/cachekit/serializers/base.py b/src/cachekit/serializers/base.py index 6b44cbfb..f076b0a0 100644 --- a/src/cachekit/serializers/base.py +++ b/src/cachekit/serializers/base.py @@ -8,6 +8,10 @@ from enum import Enum from typing import Any, ClassVar, Protocol, runtime_checkable +import msgpack + +from cachekit._rust_serializer import check_msgpack_structure + @runtime_checkable class SerializerProtocol(Protocol): @@ -320,3 +324,80 @@ class SuspiciousCacheEntryError(SerializationError): """ pass + + +# --------------------------------------------------------------------------- +# Owned untrusted-decode bounds (LAB-2503; protocol spec/interop-mode.md → Decode bounds) +# --------------------------------------------------------------------------- + +#: cachekit's own nesting ceiling, enforced by the Rust ``check_msgpack_structure`` +#: walk before msgpack-python ever sees the document. Two constraints pin it: the +#: protocol requires every SDK's bound to sit in 32..=1024, and it must not exceed +#: msgpack-python's C unpacker stack (a document at the ceiling has to decode after +#: passing the walk; tests/unit/protocol/test_decode_bounds.py checks exactly that). +MSGPACK_MAX_NESTING = 1024 + +#: Everything a corrupted or forged payload can make a decode raise, for serializers +#: to turn into ``SerializationError``. msgpack's own errors are ``ValueError`` +#: subclasses; the AutoSerializer object hook and the NumPy/DataFrame/Series +#: reconstructors add ``TypeError`` (``np.frombuffer`` on a forged dtype string), +#: ``OverflowError`` (a forged dict dtype whose itemsize is past C long), ``SyntaxError`` +#: (numpy's comma-string dtype parser runs ``ast.literal_eval`` on a forged shape prefix +#: such as ``"(1,f8"``), ``KeyError`` / ``AttributeError`` (indexing a dict that is not +#: the shape they wrote); ``BufferError`` is a non-u8 buffer exporter rejected at the +#: PyO3 boundary (LAB-770). Anything else — above all ``RuntimeError`` for a missing +#: optional dependency — is an environment fault, not a bad cache entry, and must bubble. +PAYLOAD_DECODE_ERRORS = (ValueError, TypeError, KeyError, AttributeError, OverflowError, BufferError, SyntaxError) + + +def unpackb_bounded(data: bytes | bytearray | memoryview, **unpack_opts: Any) -> Any: + """Decode one untrusted MessagePack document under cachekit-owned bounds. + + Why not plain ``msgpack.unpackb``: a collection header costs 1-5 bytes but may + declare up to 2**32-1 elements, and the C unpacker pre-allocates the container + (``PyList_New(n)``) *before* decoding the children. Nested headers stack those + allocations depth-first, so the library's per-collection default cap + (``max_*_len = len(data)``) still permits ~8 x 1024 x len(data) bytes of + transient heap — measured 10 KB -> 67 MB. + + The bound is the Rust extension's zero-copy, header-only walk + (``check_msgpack_structure``, documented there) run before the decode. It + rejects a document that nests deeper than :data:`MSGPACK_MAX_NESTING` or whose + open headers declare more elements or bytes than the remaining input can back. + Every element that survives is backed by >= 1 input byte, so the real decode's + total container pre-allocation is bounded by len(data) rather than by + depth x declared length. The explicit ``max_*_len=len(data)`` caps on + ``unpackb`` are unreachable once the walk passes; they are defence in depth + against a walk regression, not an independent bound. + + Every rejection is a ``ValueError`` (``FormatError``, ``ExtraData``, or the + walk's own ``ValueError`` naming the violated bound), which the read paths + already turn into a controlled cache miss. Trailing bytes are still rejected + by ``unpackb`` itself. + + Examples: + >>> unpackb_bounded(msgpack.packb({"a": [1, 2]}), raw=False) + {'a': [1, 2]} + >>> unpackb_bounded(b"\\xdc\\x07\\xd0" * 5000) # 15 KB nested-header bomb + Traceback (most recent call last): + ... + ValueError: Unpack failed: MessagePack document declares more elements than the input can back + >>> unpackb_bounded(b"\\x91" * 1025 + b"\\xc0") # one level past the ceiling + Traceback (most recent call last): + ... + ValueError: Unpack failed: MessagePack document nests deeper than 1024 levels + """ + if isinstance(data, memoryview): + # The walk (PyBuffer) and the decode must see one flat byte string: cast("B") flattens a + # multi-dimensional view and retypes any C-contiguous format ("b"/"c"/"H"...) to unsigned bytes + # without copying, so len(data) is the byte count the max_*_len caps need; a non-contiguous + # view has no flat form and is copied. + data = data.cast("B") if data.c_contiguous else bytes(data) + if not isinstance(data, bytes) and not (isinstance(data, memoryview) and isinstance(data.obj, bytes)): + # A mutable exporter (bytearray, a memoryview over one) could change between the walk and + # the decode, so both must see one immutable document. bytes and a memoryview of bytes stay + # zero-copy — the same containment proof the Rust side's bytes_view uses. + data = bytes(data) + n = len(data) + check_msgpack_structure(data, MSGPACK_MAX_NESTING) + return msgpack.unpackb(data, max_str_len=n, max_bin_len=n, max_array_len=n, max_map_len=n, max_ext_len=n, **unpack_opts) diff --git a/src/cachekit/serializers/standard_serializer.py b/src/cachekit/serializers/standard_serializer.py index 404c7797..fce1e141 100644 --- a/src/cachekit/serializers/standard_serializer.py +++ b/src/cachekit/serializers/standard_serializer.py @@ -27,7 +27,7 @@ from cachekit._rust_serializer import ByteStorage -from .base import SerializationError, SerializationFormat, SerializationMetadata +from .base import PAYLOAD_DECODE_ERRORS, SerializationError, SerializationFormat, SerializationMetadata, unpackb_bounded # Error message constants for unsupported types (Task 2) NUMPY_ERROR_MESSAGE = ( @@ -339,14 +339,11 @@ def deserialize(self, data: bytes | memoryview, metadata: SerializationMetadata msgpack_data = data # Deserialize MessagePack - return msgpack.unpackb(msgpack_data, **self._msgpack_unpack_opts) + return unpackb_bounded(msgpack_data, **self._msgpack_unpack_opts) except SerializationError: # Re-raise SerializationError (integrity check failure) without swallowing raise - except (msgpack.exceptions.UnpackException, ValueError, TypeError, BufferError) as e: - # BufferError: a non-u8 buffer exporter (e.g. numpy float array) rejected at the - # PyO3 boundary. Pre-LAB-770 bytes() coerced these to raw bytes and envelope - # validation rejected the garbage as ValueError; same contract, new cause. + except PAYLOAD_DECODE_ERRORS as e: raise SerializationError(f"Failed to deserialize MessagePack data: {e}") from e diff --git a/tests/unit/protocol/fixtures/decode-bounds.json b/tests/unit/protocol/fixtures/decode-bounds.json new file mode 100644 index 00000000..21b9180d --- /dev/null +++ b/tests/unit/protocol/fixtures/decode-bounds.json @@ -0,0 +1,258 @@ +{ + "version": "1.0.0", + "spec": "spec/interop-mode.md#decode-bounds", + "generator": "tools/decode-bounds-reference.py generate (CPython stdlib)", + "scope": "Any untrusted MessagePack decode in any SDK: interop/v1 values, the ByteStorage envelope bytes before StorageEnvelope is materialised, auto-mode payloads after the envelope is unwrapped, invalidation events. The bytes are plain MessagePack with no envelope.", + "rules": { + "depth": "Readers MUST bound nesting depth. The bound MUST be >= 32 and MUST be <= 1024; every reject vector tagged 'depth' nests deeper than 1024.", + "overclaim": "Readers MUST NOT pre-allocate for a collection/str/bin header more than the remaining input can back (each element or byte needs >= 1 input byte), and MUST reject a structurally incomplete document. Every reject vector tagged 'overclaim' has declared_slots > input_len - 1 (the root header is the only byte that is not an element). A map pair counts as two slots (key + value). Every per-header term and the running sum MUST be computed in >= 64 bits or with checked/saturating arithmetic; an overflow is itself a rejection.", + "failure_mode": "Rejection MUST surface as a catchable decode error that the SDK read path turns into a cache miss (fail-closed), never an uncaught crash or an OOM abort." + }, + "field_notes": { + "construction": "input = bytes.fromhex(repeat_hex) * count + bytes.fromhex(suffix_hex)", + "nesting_depth": "collection headers along the deepest spine (str/bin count as 0)", + "declared_slots": "sum of every header's declared element/byte count; a map pair counts as two slots (key + value); a nested header counts as one element of its parent", + "reject_reasons": "which rule(s) the vector violates; a maintainer note, not a normative message" + }, + "reject_vectors": [ + { + "name": "nested_array16_depth_2048", + "description": "2048 nested array16 headers each claiming 2000 elements, 0 backing bytes. The LAB-2487 amplifier shape: an eager decoder pre-allocates 2000 slots per level before hitting EOF. 2000 < input_len, so a per-collection cap of len(input) does NOT reject it.", + "construction": { + "repeat_hex": "dc07d0", + "count": 2048, + "suffix_hex": "" + }, + "input_hex": "dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0dc07d0", + "input_len": 6144, + "nesting_depth": 2048, + "declared_slots": 4096000, + "reject_reasons": [ + "depth", + "overclaim" + ] + }, + { + "name": "nested_array32_input_len_depth_1100", + "description": "1100 nested array32 headers each claiming exactly len(input)=5500 elements. Defeats a per-collection cap of len(input): peak pre-allocation is depth x len(input) x slot size.", + "construction": { + "repeat_hex": "dd0000157c", + "count": 1100, + "suffix_hex": "" + }, + "input_hex": "dd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157cdd0000157c", + "input_len": 5500, + "nesting_depth": 1100, + "declared_slots": 6050000, + "reject_reasons": [ + "depth", + "overclaim" + ] + }, + { + "name": "nested_map16_depth_2048", + "description": "Map twin of nested_array16_depth_2048 (map pre-allocation is typically larger per slot).", + "construction": { + "repeat_hex": "de07d0", + "count": 2048, + "suffix_hex": "" + }, + "input_hex": "de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0de07d0", + "input_len": 6144, + "nesting_depth": 2048, + "declared_slots": 8192000, + "reject_reasons": [ + "depth", + "overclaim" + ] + }, + { + "name": "nested_fixarray_depth_2048_complete", + "description": "Structurally COMPLETE document ([[...[null]...]]) nested 2048 deep: every header is backed, so only the depth bound rejects it. Isolates the depth rule from the allocation rule.", + "construction": { + "repeat_hex": "91", + "count": 2048, + "suffix_hex": "c0" + }, + "input_hex": "9191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191919191c0", + "input_len": 2049, + "nesting_depth": 2048, + "declared_slots": 2048, + "reject_reasons": [ + "depth" + ] + }, + { + "name": "array16_overclaim_shallow", + "description": "One array16 header claiming 10 000 elements with 3 backing bytes.", + "construction": { + "repeat_hex": "dc2710", + "count": 1, + "suffix_hex": "010203" + }, + "input_hex": "dc2710010203", + "input_len": 6, + "nesting_depth": 1, + "declared_slots": 10000, + "reject_reasons": [ + "overclaim" + ] + }, + { + "name": "array32_max_claim_alone", + "description": "A lone 5-byte array32 header claiming 2^32-1 elements.", + "construction": { + "repeat_hex": "ddffffffff", + "count": 1, + "suffix_hex": "" + }, + "input_hex": "ddffffffff", + "input_len": 5, + "nesting_depth": 1, + "declared_slots": 4294967295, + "reject_reasons": [ + "overclaim" + ] + }, + { + "name": "map32_max_claim_alone", + "description": "A lone 5-byte map32 header claiming 2^32-1 pairs (2^33-2 slots: each pair is a key and a value).", + "construction": { + "repeat_hex": "dfffffffff", + "count": 1, + "suffix_hex": "" + }, + "input_hex": "dfffffffff", + "input_len": 5, + "nesting_depth": 1, + "declared_slots": 8589934590, + "reject_reasons": [ + "overclaim" + ] + }, + { + "name": "array32_sum_wraps_u32", + "description": "array32 claiming 2^32-1 elements whose first element is an array32 claiming 1: the declared slots sum to exactly 2^32, which a 32-bit accumulator wraps to 0 and then passes the slot budget.", + "construction": { + "repeat_hex": "ddffffffff", + "count": 1, + "suffix_hex": "dd00000001" + }, + "input_hex": "ddffffffffdd00000001", + "input_len": 10, + "nesting_depth": 2, + "declared_slots": 4294967296, + "reject_reasons": [ + "overclaim" + ] + }, + { + "name": "map32_half_claim_wraps_u32_mul", + "description": "A lone map32 header claiming 2^31 pairs: the per-header term 2 x pairs is exactly 2^32, which a 32-bit multiply wraps to 0 before it is ever added to the budget.", + "construction": { + "repeat_hex": "df80000000", + "count": 1, + "suffix_hex": "" + }, + "input_hex": "df80000000", + "input_len": 5, + "nesting_depth": 1, + "declared_slots": 4294967296, + "reject_reasons": [ + "overclaim" + ] + }, + { + "name": "fixmap_short_by_one", + "description": "fixmap claiming 1 pair with the key present and the value missing: the map twin of fixarray_short_by_one. Counting one slot per pair (instead of two) accepts it.", + "construction": { + "repeat_hex": "81", + "count": 1, + "suffix_hex": "c0" + }, + "input_hex": "81c0", + "input_len": 2, + "nesting_depth": 1, + "declared_slots": 2, + "reject_reasons": [ + "overclaim" + ] + }, + { + "name": "bin32_overclaim", + "description": "bin32 header claiming 2^32-1 bytes with 1 backing byte (a 6-byte document declaring a 4 GiB buffer).", + "construction": { + "repeat_hex": "c6ffffffff", + "count": 1, + "suffix_hex": "41" + }, + "input_hex": "c6ffffffff41", + "input_len": 6, + "nesting_depth": 0, + "declared_slots": 4294967295, + "reject_reasons": [ + "overclaim" + ] + }, + { + "name": "str32_overclaim", + "description": "str32 twin of bin32_overclaim.", + "construction": { + "repeat_hex": "dbffffffff", + "count": 1, + "suffix_hex": "41" + }, + "input_hex": "dbffffffff41", + "input_len": 6, + "nesting_depth": 0, + "declared_slots": 4294967295, + "reject_reasons": [ + "overclaim" + ] + }, + { + "name": "fixarray_short_by_one", + "description": "fixarray claiming 5 elements with 4 present: the minimal truncated document.", + "construction": { + "repeat_hex": "95", + "count": 1, + "suffix_hex": "c0c0c0c0" + }, + "input_hex": "95c0c0c0c0", + "input_len": 5, + "nesting_depth": 1, + "declared_slots": 5, + "reject_reasons": [ + "overclaim" + ] + } + ], + "accept_vectors": [ + { + "name": "nested_fixarray_depth_32", + "description": "[[...[null]...]] nested 32 deep, complete. A conforming reader MUST accept it: the depth bound may not be tighter than 32.", + "construction": { + "repeat_hex": "91", + "count": 32, + "suffix_hex": "c0" + }, + "input_hex": "9191919191919191919191919191919191919191919191919191919191919191c0", + "input_len": 33, + "nesting_depth": 32, + "declared_slots": 32 + }, + { + "name": "array16_256_backed_nils", + "description": "array16 header claiming 256 elements with all 256 present. A *16 header that is fully backed by input is legitimate; the allocation rule is about backing, not header width.", + "construction": { + "repeat_hex": "dc0100", + "count": 1, + "suffix_hex": "c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0" + }, + "input_hex": "dc0100c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0", + "input_len": 259, + "nesting_depth": 1, + "declared_slots": 256 + } + ] +} diff --git a/tests/unit/protocol/test_decode_bounds.py b/tests/unit/protocol/test_decode_bounds.py new file mode 100644 index 00000000..786321d4 --- /dev/null +++ b/tests/unit/protocol/test_decode_bounds.py @@ -0,0 +1,242 @@ +"""Untrusted-decode bounds (LAB-2503): protocol vectors + the SDK-local regression guard. + +Why the bound exists and how it works: the ``unpackb_bounded`` docstring in +``cachekit.serializers.base`` (the canonical home). This file pins, so a +msgpack-python bump cannot silently move it: +- every reject vector is rejected on every decode path, with a bounded peak; +- every accept vector decodes on every path (the bound cannot over-tighten); +- the nesting ceiling is exactly MSGPACK_MAX_NESTING; +- the read path turns a bomb into SerializationError (a controlled miss), not a crash. + +Fixture: tests/unit/protocol/fixtures/decode-bounds.json, vendored from +cachekit-io/protocol test-vectors/decode-bounds.json (sha256 pinned below). +Regenerate ONLY by re-copying from the protocol repo — never by hand. +""" + +from __future__ import annotations + +import functools +import hashlib +import json +import tracemalloc +from collections.abc import Callable +from pathlib import Path +from typing import Any + +import msgpack +import pytest + +from cachekit._rust_serializer import ByteStorage, check_msgpack_structure +from cachekit.cache_handler import CacheSerializationHandler +from cachekit.interop import decode_interop_value +from cachekit.serializers.auto_serializer import AutoSerializer +from cachekit.serializers.base import MSGPACK_MAX_NESTING, SerializationError, unpackb_bounded +from cachekit.serializers.standard_serializer import StandardSerializer +from cachekit.serializers.wrapper import SerializationWrapper + +FIXTURE_PATH = Path(__file__).parent / "fixtures" / "decode-bounds.json" +FIXTURE_SHA256 = "75c1204e6f58f5220581d3e40e75a68f2df605b4e3c817107b0c690cd7da5cd4" # pragma: allowlist secret +VECTORS = json.loads(FIXTURE_PATH.read_text(encoding="utf-8")) +EXPECTED_COUNTS = {"reject_vectors": 13, "accept_vectors": 2} + +# Peak transient heap a rejected decode may cost: a small constant (tracemalloc + unpackb +# overhead) plus a few multiples of the input. Unguarded, the nested_array32_input_len vector +# peaks at ~8000x its input, so this discriminates by three orders of magnitude. +PEAK_BUDGET = 2 * 1024 * 1024 +PEAK_PER_INPUT_BYTE = 4 + + +def _envelope(payload: bytes) -> bytes: + return bytes(ByteStorage("msgpack").store(payload, "msgpack")) + + +CACHE_KEY = "ns:decode:bounds" + + +@functools.lru_cache(maxsize=2) +def _frame_template(serializer: str = "default") -> tuple[dict[str, Any], str]: + _, metadata, serializer_name = SerializationWrapper.unwrap( + CacheSerializationHandler(serializer).serialize_data({"t": 1}, cache_key=CACHE_KEY) + ) + return metadata, serializer_name + + +def _forged_entry(payload: bytes) -> bytes: + """A genuine CK v3 frame with its payload swapped — the backend-write attacker's move.""" + metadata, serializer_name = _frame_template() + return SerializationWrapper.wrap(_envelope(payload), metadata, serializer_name) + + +# Every path that decodes backend-supplied MessagePack. Each must reach unpackb_bounded. +DECODE_PATHS: dict[str, Callable[[bytes], Any]] = { + "unpackb_bounded": lambda b: unpackb_bounded(b, raw=False), + "interop": decode_interop_value, + "standard/plain": StandardSerializer(enable_integrity_checking=False).deserialize, + "standard/envelope": lambda b: StandardSerializer().deserialize(_envelope(b)), + "auto/plain": AutoSerializer(enable_integrity_checking=False).deserialize, + "auto/envelope": lambda b: AutoSerializer().deserialize(_envelope(b)), + "handler.deserialize_data": lambda b: CacheSerializationHandler().deserialize_data(_forged_entry(b), cache_key=CACHE_KEY), +} + + +def _peak_of(fn: Callable[..., Any], *args: Any) -> tuple[Any, BaseException | None, int]: + tracemalloc.start() + try: + return fn(*args), None, tracemalloc.get_traced_memory()[1] + except (ValueError, SerializationError) as e: + # The only rejections the read path maps to a controlled miss; any other type propagates. + return None, e, tracemalloc.get_traced_memory()[1] + finally: + tracemalloc.stop() + + +def _vector_ids(group: str) -> list[str]: + return [v["name"] for v in VECTORS[group]] + + +def _reject_vector(name: str) -> bytes: + return bytes.fromhex(next(v["input_hex"] for v in VECTORS["reject_vectors"] if v["name"] == name)) + + +class TestFixtureIsTheVendoredProtocolFile: + def test_sha256_and_counts(self) -> None: + assert hashlib.sha256(FIXTURE_PATH.read_bytes()).hexdigest() == FIXTURE_SHA256 + assert {g: len(VECTORS[g]) for g in EXPECTED_COUNTS} == EXPECTED_COUNTS + assert VECTORS["spec"] == "spec/interop-mode.md#decode-bounds" + + +@pytest.mark.parametrize("path", DECODE_PATHS) +class TestProtocolVectors: + @pytest.mark.parametrize("vector", VECTORS["reject_vectors"], ids=_vector_ids("reject_vectors")) + def test_reject_vector_is_rejected_with_bounded_peak(self, path: str, vector: dict[str, Any]) -> None: + data = bytes.fromhex(vector["input_hex"]) + _, err, peak = _peak_of(DECODE_PATHS[path], data) + assert err is not None, f"{vector['name']}: {path} decoded a reject vector" + assert peak < PEAK_BUDGET + PEAK_PER_INPUT_BYTE * len(data), f"{vector['name']}: {path} peaked at {peak} bytes" + + @pytest.mark.parametrize("vector", VECTORS["accept_vectors"], ids=_vector_ids("accept_vectors")) + def test_accept_vector_decodes(self, path: str, vector: dict[str, Any]) -> None: + data = bytes.fromhex(vector["input_hex"]) + value = DECODE_PATHS[path](data) + depth = 0 + while isinstance(value, list): + depth, value = depth + 1, value[0] if value else None + assert depth == vector["nesting_depth"] + + +class TestOwnedBounds: + """SDK-local guards that go beyond the shared vectors.""" + + def test_nesting_ceiling_is_exactly_the_pinned_constant(self) -> None: + # The walk rejects one level past MSGPACK_MAX_NESTING; a document AT the ceiling + # must still decode, so the constant may not exceed msgpack-python's C stack. + at_bound = b"\x91" * MSGPACK_MAX_NESTING + b"\xc0" + assert unpackb_bounded(at_bound) == json.loads("[" * MSGPACK_MAX_NESTING + "null" + "]" * MSGPACK_MAX_NESTING) + with pytest.raises(ValueError, match=f"nests deeper than {MSGPACK_MAX_NESTING} levels"): + unpackb_bounded(b"\x91" * (MSGPACK_MAX_NESTING + 1) + b"\xc0") + + def test_trailing_bytes_still_rejected(self) -> None: + with pytest.raises(msgpack.exceptions.ExtraData): + unpackb_bounded(b"\xc0\xc0") + + def test_mutable_exporters_are_accepted(self) -> None: + # A bytearray (or a memoryview over one) is snapshotted so the walk and the decode see one + # immutable document; a memoryview of bytes stays zero-copy. All three must decode. + doc = msgpack.packb({"t": 1}) + assert unpackb_bounded(bytearray(doc), raw=False) == {"t": 1} + assert unpackb_bounded(memoryview(bytearray(doc)), raw=False) == {"t": 1} + assert unpackb_bounded(memoryview(doc)[0:], raw=False) == {"t": 1} + + def test_memoryview_shapes_and_formats_decode_like_bytes(self) -> None: + # The Rust walk takes PyBuffer; msgpack takes any itemsize-1 buffer. Views are normalised + # to a flat "B" view first so the two agree, len(data) is the byte count the caps need, and a + # legitimate document is never rejected for the shape or format of the view it arrived in. + doc = next( + d + for d in (msgpack.packb({"k": b"x" * m, "l": [1, 2, 3]}, use_bin_type=True) for m in range(1, 9)) + if len(d) % 8 == 0 + ) + expected = msgpack.unpackb(doc, raw=False) + views = { + "signed char": memoryview(doc).cast("b"), + "char": memoryview(doc).cast("c"), + "2-D bytes": memoryview(doc).cast("B", shape=[len(doc) // 8, 8]), + "uint16": memoryview(doc).cast("H"), + } + for name, view in views.items(): + assert unpackb_bounded(view, raw=False) == expected, name + # A non-contiguous view has no flat form: it is copied, then decoded like the bytes it selects. + interleaved = bytes(b for pair in zip(doc, doc, strict=True) for b in pair) + assert unpackb_bounded(memoryview(interleaved)[::2], raw=False) == expected + + # One exact-width document per fixed-width marker family: float32/64, uint8..64, int8..64, + # fixext 1/2/4/8/16, ext8/16/32 (2-byte payload), str8/16/32 + fixstr, bin8/16/32. + FIXED_WIDTH_DOCS = [ + b"\xca" + b"\x00" * 4, + b"\xcb" + b"\x00" * 8, + b"\xcc\x00", + b"\xcd\x00\x00", + b"\xce" + b"\x00" * 4, + b"\xcf" + b"\x00" * 8, + b"\xd0\x00", + b"\xd1\x00\x00", + b"\xd2" + b"\x00" * 4, + b"\xd3" + b"\x00" * 8, + b"\xd4\x01\x00", + b"\xd5\x01\x00\x00", + b"\xd6\x01" + b"\x00" * 4, + b"\xd7\x01" + b"\x00" * 8, + b"\xd8\x01" + b"\x00" * 16, + b"\xc7\x02\x01\x00\x00", + b"\xc8\x00\x02\x01\x00\x00", + b"\xc9\x00\x00\x00\x02\x01\x00\x00", + b"\xa1x", + b"\xd9\x01x", + b"\xda\x00\x01x", + b"\xdb\x00\x00\x00\x01x", + b"\xc4\x01x", + b"\xc5\x00\x01x", + b"\xc6\x00\x00\x00\x01x", + ] + + @pytest.mark.parametrize("doc", FIXED_WIDTH_DOCS, ids=lambda d: f"0x{d[0]:02x}") + def test_every_marker_is_walked_to_its_exact_width(self, doc: bytes) -> None: + # Exact length passes the walk; one byte short is a truncation; a trailing byte reaches the + # decoder as ExtraData — together they pin that the walk consumed exactly the marker's width. + check_msgpack_structure(doc, MSGPACK_MAX_NESTING) + with pytest.raises(ValueError, match="Unpack failed"): + check_msgpack_structure(doc[:-1], MSGPACK_MAX_NESTING) + with pytest.raises(msgpack.exceptions.ExtraData): + unpackb_bounded(doc + b"\xc0") + + def test_reserved_marker_is_rejected(self) -> None: + with pytest.raises(ValueError, match="reserved marker 0xc1"): + check_msgpack_structure(b"\xc1", MSGPACK_MAX_NESTING) + + def test_plain_path_miss_does_not_depend_on_numpy(self, monkeypatch: pytest.MonkeyPatch) -> None: + # Without the [data] extra (the free-threaded CI lane) a forged plain entry must still be a + # SerializationError — not the RuntimeError a NumPy fallback raises for a missing numpy. + monkeypatch.setattr("cachekit.serializers.auto_serializer.HAS_NUMPY", False) + with pytest.raises(SerializationError, match="not a decodable MessagePack payload"): + AutoSerializer(enable_integrity_checking=False).deserialize(_reject_vector("bin32_overclaim")) + + def test_validate_data_reports_a_bomb_as_invalid_within_the_peak_budget(self) -> None: + # Python-only validate_data is a decode path too: a bomb must read as invalid (not raise), + # and the walk must have stopped it before the decoder pre-allocated ~8000x the input. + serializer = AutoSerializer(enable_integrity_checking=False) + assert serializer.validate_data(msgpack.packb({"t": 1})) is True + bomb = _reject_vector("nested_array32_input_len_depth_1100") + valid, err, peak = _peak_of(serializer.validate_data, bomb) + assert (valid, err) == (False, None) + assert peak < PEAK_BUDGET + PEAK_PER_INPUT_BYTE * len(bomb), f"validate_data peaked at {peak} bytes" + + @pytest.mark.parametrize("original_type", ["dataframe", "series"]) + def test_bomb_behind_a_dataframe_or_series_frame_is_a_controlled_miss(self, original_type: str) -> None: + # AutoSerializer's metadata routes decode outside the verified-envelope normaliser; the bound's + # rejection must still reach the handler as SerializationError (evict + tamper hook), never a + # bare ValueError. The message match keeps a "Serializer mismatch" error from faking a pass. + metadata, serializer_name = _frame_template("auto") + bomb = _reject_vector("nested_array32_input_len_depth_1100") + frame = SerializationWrapper.wrap(_envelope(bomb), {**metadata, "original_type": original_type}, serializer_name) + with pytest.raises(SerializationError, match=f"failed to decode as {original_type}"): + CacheSerializationHandler("auto").deserialize_data(frame, cache_key=CACHE_KEY) diff --git a/tests/unit/test_auto_serializer_mutation_and_corruption.py b/tests/unit/test_auto_serializer_mutation_and_corruption.py index 8eb239ed..bfb41f30 100644 --- a/tests/unit/test_auto_serializer_mutation_and_corruption.py +++ b/tests/unit/test_auto_serializer_mutation_and_corruption.py @@ -12,12 +12,20 @@ DataFrames route through ArrowSerializer when pyarrow is installed, so the columnar msgpack path (``_serialize_dataframe`` / the ``"dataframe"`` branch) is exercised by disabling the arrow serializer. Series never use arrow, so they hit the columnar path unconditionally. + +LAB-2503: ``TestDataFrameSeriesReadRoutes`` pins every DataFrame/Series read route (metadata x +integrity, and metadata-less via the envelope's format_id); it lives here because this file +already forces the columnar path. """ from __future__ import annotations +import functools + +import msgpack import pytest +from cachekit._rust_serializer import ByteStorage from cachekit.serializers import AutoSerializer from cachekit.serializers.base import SerializationError @@ -27,13 +35,45 @@ pd = pytest.importorskip("pandas") -def _no_arrow() -> AutoSerializer: +def _no_arrow(**kwargs: bool) -> AutoSerializer: """An AutoSerializer forced onto the columnar msgpack DataFrame path (pyarrow absent).""" - s = AutoSerializer() + s = AutoSerializer(**kwargs) s._arrow_serializer = None return s +def _assert_equal(out: pd.DataFrame | pd.Series, expected: pd.DataFrame | pd.Series) -> None: + if isinstance(expected, pd.DataFrame): + pd.testing.assert_frame_equal(out, expected) + else: + pd.testing.assert_series_equal(out, expected) + + +FRAME = pd.DataFrame({"x": np.arange(5, dtype=np.float64), "n": np.arange(5, dtype=np.int64)}) +SERIES = pd.Series(np.arange(8, dtype=np.float64), name="v") + + +@pytest.mark.unit +class TestDataFrameSeriesReadRoutes: + """Every route a DataFrame/Series read can take must reconstruct the value: with metadata + on both integrity settings, and — the decorator read path may carry none — from the + verified envelope's own format_id (LAB-2503 moved that route under the fail-closed guard). + """ + + @pytest.mark.parametrize("value", [FRAME, SERIES], ids=["dataframe", "series"]) + @pytest.mark.parametrize("integrity", [True, False], ids=["integrity-on", "integrity-off"]) + def test_roundtrip_with_metadata(self, value: pd.DataFrame | pd.Series, integrity: bool) -> None: + s = _no_arrow(enable_integrity_checking=integrity) + data, meta = s.serialize(value) + _assert_equal(s.deserialize(data, meta), value) + + @pytest.mark.parametrize("value", [FRAME, SERIES], ids=["dataframe", "series"]) + def test_roundtrip_without_metadata_via_envelope_format_id(self, value: pd.DataFrame | pd.Series) -> None: + s = _no_arrow() + data, _ = s.serialize(value) + _assert_equal(s.deserialize(data), value) + + @pytest.mark.unit class TestDeserializedArraysAreWritable: """#157: deserialized numeric arrays must be writable and must not alias the cached buffer.""" @@ -94,3 +134,109 @@ def test_dataframe_corruption_raises_serialization_error(self) -> None: corrupted[len(corrupted) // 2] ^= 0xFF with pytest.raises(SerializationError): s.deserialize(bytes(corrupted), meta) + + +# A well-formed __ndarray__ marker: the object hook turns it into an ndarray wherever it sits, so a +# forged document can put an array where the writer only ever puts a list or a dict. M8[2s] is a +# dtype numpy accepts and pandas then asserts on (AssertionError, outside PAYLOAD_DECODE_ERRORS). +NDARRAY_M8_2S = {"__ndarray__": True, "dtype": "M8[2s]", "shape": [1], "data": b"\x00" * 8} +F8_COLUMN = {"type": "numeric", "data": b"\x00" * 8, "dtype": " bytes: + """A checksummed ``dataframe`` / ``series`` entry carrying ``body``.""" + return bytes(ByteStorage("msgpack").store(msgpack.packb(body), kind)) + + +def _columnar_entry(kind: str, column: dict) -> bytes: + """A checksummed ``dataframe`` / ``series`` entry whose single column is ``column``.""" + body = ( + {"columns": ["x"], "index": None, "data": {"x": column}} + if kind == "dataframe" + else {"name": None, "index": None, **column} + ) + return _entry(kind, body) + + +@pytest.mark.unit +class TestForgedColumnarPayloadIsRefused: + """Forged DataFrame/Series documents are refused before pandas sees them (LAB-2503): a numeric + column dtype the writer never emits (``M8[0ns]`` passes ``np.frombuffer`` and then kills the + process with SIGFPE inside pandas — uncatchable), a column type marker other than the two the + writer emits, and an ndarray smuggled via the ``__ndarray__`` hook into a field the writer only + ever fills with a list or a dict. + """ + + @pytest.mark.parametrize("dtype", ["M8[0ns]", "m8[0ns]", "U4"]) + @pytest.mark.parametrize("kind", ["dataframe", "series"]) + def test_forged_column_dtype_is_a_serialization_error(self, kind: str, dtype: str) -> None: + entry = _columnar_entry(kind, {**F8_COLUMN, "dtype": dtype}) + with pytest.raises(SerializationError, match="Forged columnar dtype"): + AutoSerializer().deserialize(entry) + + @pytest.mark.parametrize("marker", ["forged", DEEP_LIST], ids=["unknown-string", "list-nested-1000-deep"]) + @pytest.mark.parametrize("kind", ["dataframe", "series"]) + def test_unknown_column_type_marker_is_refused(self, kind: str, marker: object) -> None: + entry = _columnar_entry(kind, {"type": marker, "data": [1, 2]}) + with pytest.raises(SerializationError, match="Forged columnar payload: .* type is"): + AutoSerializer().deserialize(entry) + + @pytest.mark.parametrize( + "kind, body", + [ + ("dataframe", {"columns": ["x"], "index": None, "data": {"x": NDARRAY_M8_2S}}), + ("dataframe", {"columns": ["x"], "index": None, "data": {"x": {"type": "object", "data": NDARRAY_M8_2S}}}), + ("dataframe", {"columns": NDARRAY_M8_2S, "index": None, "data": {"x": F8_COLUMN}}), + ("dataframe", {"columns": ["x"], "index": NDARRAY_M8_2S, "data": {"x": F8_COLUMN}}), + ("dataframe", {"columns": ["x"], "index": None, "data": NDARRAY_M8_2S}), + ("series", {"name": None, "index": None, "type": "object", "data": NDARRAY_M8_2S}), + ("series", {"name": None, "index": NDARRAY_M8_2S, **F8_COLUMN}), + ], + ids=[ + "df-column-is-ndarray", + "df-object-data-is-ndarray", + "df-columns-is-ndarray", + "df-index-is-ndarray", + "df-data-is-ndarray", + "series-object-data-is-ndarray", + "series-index-is-ndarray", + ], + ) + def test_ndarray_where_the_writer_emits_a_list_or_dict_is_refused(self, kind: str, body: dict) -> None: + with pytest.raises(SerializationError, match="Forged columnar payload"): + AutoSerializer().deserialize(_entry(kind, body)) + + +def _numpy_raw(dtype: bytes, shape: bytes) -> bytes: + return b"NUMPY_RAW" + len(dtype).to_bytes(2, "little") + dtype + len(shape).to_bytes(2, "little") + shape + + +class TestForgedNumpyMetadataIsRefused: + """NUMPY_RAW dtype/shape metadata is untrusted: slicing past the end silently shortens and a + partial 4-byte chunk used to parse as a dimension, so a forged 1-byte zero shape chunk built + an EMPTY array instead of raising (CodeRabbit on cachekit-py#276).""" + + @pytest.mark.parametrize( + ("payload", "why"), + [ + (_numpy_raw(b" empty array"), + (_numpy_raw(b" None: + pytest.importorskip("numpy") + with pytest.raises(SerializationError, match="truncated or misaligned"): + AutoSerializer(enable_integrity_checking=False).deserialize(payload) + + def test_well_formed_numpy_still_round_trips(self) -> None: + np = pytest.importorskip("numpy") + arr = np.arange(6, dtype=" None: + with pytest.raises(SerializationError): + serializer.deserialize(wrap(payload)) + + +def test_hook_diagnostic_propagates_unwrapped_from_the_verified_envelope() -> None: + """The object hook's SerializationError sits outside PAYLOAD_DECODE_ERRORS, so it leaves the + verified-envelope decode unwrapped — that catch must never widen back to ``Exception``.""" + entry = bytes(ByteStorage("msgpack").store(msgpack.packb({"__uuid__": True}), "msgpack")) + with pytest.raises(SerializationError, match=r"^Invalid UUID format: missing 'value' field") as excinfo: + AutoSerializer().deserialize(entry) + assert excinfo.value.__cause__ is None + class TestAutoSerializerUUID: """Test UUID serialization support.""" diff --git a/tests/unit/test_auto_serializer_numpy_integrity.py b/tests/unit/test_auto_serializer_numpy_integrity.py index 4644a842..47729e30 100644 --- a/tests/unit/test_auto_serializer_numpy_integrity.py +++ b/tests/unit/test_auto_serializer_numpy_integrity.py @@ -177,3 +177,43 @@ def test_numpy_roundtrip_through_encryption(self) -> None: result = wrapper.deserialize(data, metadata, cache_key) np.testing.assert_array_equal(result, original) + + +def _numpy_raw(dtype: bytes, shape: tuple[int, ...], payload: bytes) -> bytes: + """A NUMPY_RAW entry laid out exactly as ``_serialize_numpy`` writes it, fields attacker-chosen.""" + shape_data = b"".join(dim.to_bytes(4, "little") for dim in shape) + header = len(dtype).to_bytes(2, "little") + dtype + len(shape_data).to_bytes(2, "little") + shape_data + return b"NUMPY_RAW" + header + payload + + +# Each passes the header checks and reaches _deserialize_numpy's own except clause — the dtype +# decode or numpy raising TypeError / ValueError / SyntaxError (measured on numpy 1.26-2.3). +FORGED_NUMPY_RAW = { + "dtype-not-understood": _numpy_raw(b"not-a-dtype", (1,), b"\x00" * 8), + "itemsize-past-c-long": _numpy_raw(b"V9223372036854775808", (1,), b"\x00" * 8), + "dtype-not-utf8": _numpy_raw(b"\xff\xfe", (1,), b"\x00" * 8), + "shape-does-not-fit": _numpy_raw(b" ast.literal_eval -> SyntaxError +} + + +@pytest.mark.unit +class TestAutoSerializerNumpyForgedEntries: + """A NUMPY_RAW entry is routed to ``_deserialize_numpy`` structurally, with no outer + ``PAYLOAD_DECODE_ERRORS`` normalisation, so its own except clause is the whole fail-closed + contract for a forged entry (LAB-2503). The checksum is unkeyed and does not help: whoever + can write the backend can also write a matching xxHash3-64. + """ + + @pytest.mark.parametrize("entry", FORGED_NUMPY_RAW.values(), ids=list(FORGED_NUMPY_RAW)) + @pytest.mark.parametrize("checksummed", [False, True], ids=["raw", "checksummed"]) + def test_forged_entry_fails_closed_as_serialization_error(self, entry: bytes, checksummed: bool) -> None: + if checksummed: + entry = xxhash.xxh3_64_digest(entry) + entry + with pytest.raises(SerializationError, match="Failed to deserialize NumPy array"): + AutoSerializer().deserialize(entry) + + def test_degenerate_datetime_dtype_is_refused_before_any_array_is_built(self) -> None: + # M8[0ns] passes np.frombuffer and then kills the process with SIGFPE inside pandas. + with pytest.raises(SerializationError, match="zero datetime unit multiplier"): + AutoSerializer().deserialize(_numpy_raw(b"M8[0ns]", (1,), b"\x00" * 8))