diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d53ad45b713..facd40fe9c8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -24,7 +24,7 @@ repos: hooks: - id: check-added-large-files # Retain the exact lossless cross-platform performance evidence archives. - exclude: ^docs/benchmarks/((executor-copy-audit|bounded-read-ahead)-20261101|(directory-task-slots|packed-visit-snapshots|bounded-match-results|useful-match-workers|direct-evaluator-workers|derived-content-handoff|immutable-default-catalogs|prepared-diff-configuration|prepared-hash-operands|aggregate-comparison-totals|byte-shingle-representations|archive-input-buffers)-20261102|(digest-and-record-storage|caller-owned-streams)-20261103)\.json\.gz$ + exclude: ^docs/benchmarks/((executor-copy-audit|bounded-read-ahead)-20261101|(directory-task-slots|packed-visit-snapshots|bounded-match-results|useful-match-workers|direct-evaluator-workers|derived-content-handoff|immutable-default-catalogs|prepared-diff-configuration|prepared-hash-operands|aggregate-comparison-totals|byte-shingle-representations|archive-input-buffers)-20261102|(digest-and-record-storage|caller-owned-streams|direct-source-materialization)-20261103)\.json\.gz$ - id: check-merge-conflict - id: check-yaml # BINARY_TYPES below: archive containers are binary but can START with text (a phar begins with diff --git a/CHANGELOG.md b/CHANGELOG.md index d8d52105f05..79cd18f663c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,8 @@ # 0.11.0 +- Fill bounded source ranges directly into final storage and reuse owned decoder blocks during materialization, preserving read limits, errors and budgets. + - Reuse caller-owned input storage for memory-backed and nested archive-member streams while preserving read errors, sparse content and shared budgets. - Reduce cached-digest copies and plain/escaped output allocation churn while preserving snapshot lifetimes and exact output bytes. diff --git a/TODO.md b/TODO.md index eda939bc12e..b9d422043b0 100644 --- a/TODO.md +++ b/TODO.md @@ -455,9 +455,10 @@ EP/JC/P work; candidates are not speedup claims until measured on both platforms nested containers, short reads, errors, cursor interleaving and budget/owner lifetimes. Keep an explicitly incapable legacy benchmark control and separately exercise actual memory/member sources. [Qualification, costs and rejected rewrite](docs/benchmarks/caller-owned-streams-20261103.md) retain all four variants and duplicate-baseline controls. -- [ ] O19d: Qualify direct filling of materialized ranges/results through caller-owned reads, with +- [x] O19d: Qualify direct filling of materialized ranges/results through caller-owned reads, with exact overflow/limit probes, EOF, changing capabilities and owned fallback. Keep range memory reservations and replay accounting; do not silently treat a zero-length read as proof of EOF. + Keep [qualification, rejected fallback and slower controls](docs/benchmarks/direct-source-materialization-20261103.md). - [ ] O20: Qualify optional bulk metadata by backend capability with precise fallback, while keeping metadata-free traversal free of extra probes and extended-stat work. - [x] O21a: Qualify staged one-word keys, borrowed normalized shingle windows and an isolated byte diff --git a/bazelmod/clang_tidy_targets.bzl b/bazelmod/clang_tidy_targets.bzl index 1fc429e5359..5baa103406d 100644 --- a/bazelmod/clang_tidy_targets.bzl +++ b/bazelmod/clang_tidy_targets.bzl @@ -10,6 +10,7 @@ keeps the two sets identical. CLANG_TIDY_MANUAL_TARGETS = [ "//tools:archive_source_benchmark", + "//tools:read_source_benchmark", "//xff/cli:globals_benchmark", "//xff/engine:read_accounting_test", "//xff/engine:compare_benchmark", diff --git a/docs/benchmarks/direct-source-materialization-20261103.json.gz b/docs/benchmarks/direct-source-materialization-20261103.json.gz new file mode 100644 index 00000000000..fef44dd503e Binary files /dev/null and b/docs/benchmarks/direct-source-materialization-20261103.json.gz differ diff --git a/docs/benchmarks/direct-source-materialization-20261103.md b/docs/benchmarks/direct-source-materialization-20261103.md new file mode 100644 index 00000000000..f0d18bc61fe --- /dev/null +++ b/docs/benchmarks/direct-source-materialization-20261103.md @@ -0,0 +1,162 @@ +# Direct source result materialization (2026-11-03) + +## Decision + +Fill bounded ranges and supported materializations directly into final string storage. Keep the existing owned-read path for incapable decoders, adopting the first owned block when the result is empty. Later owned chunks append normally. Observe capability once per read, allowing later changes. No new public API, flag, dependency, worker, lock, cache, metadata probe or host-I/O route. + +Validate every count before exposing initialized bytes. Preserve short reads, cursor position, errors, range reservations, offset overflow and the 64 MiB range guard. Default `Size` and sequential `OpenAt` retain their replay accounting. At exact materialization limits, including zero, a positive one-byte read distinguishes EOF from overflow; errors remain errors. Clamp the logical limit to the string maximum before arithmetic. + +Supported reads reuse spare capacity, including SSO, rather than growing a string merely to discover EOF after a short read. This may add a small initial backend read for a larger input; it is included in the measurements. Final capacity can still grow geometrically. Source/backend and small controls can be slower; this is not a universal speedup. + +## Protocol + +Publication parent `e6a60c7ec0d9517e5e0bcf99bafe76b379dc32f6` adds only the full-rg integration-test timeout adjustment. Production and benchmark inputs remain byte-identical to the qualified parent; the raw evidence retains the comparison proof. + +Fixed synchronized parent `c6a32909b17ca74eb6343f8dc938067c3bad8c69` includes #1065 dependencies and #1066/#1067/#1068 runtime work. Four variants isolate baseline, range-only, bytes-only and combined changes. Common tests and benchmarks are identical. Filename-aware formatting gates precede builds; raw artifacts retain compiler hashes, release flags, link actions, source hashes, binary hashes and sizes. + +Per host: 100 native cases x three sessions over nested TAR-in-TAR/TAR-in-gzip-TAR/ZIP-in-TAR, flat archive controls and ordinary files, 10/1,000/10,000 members, 128/4,096-byte payloads, one/three requested workers and listing/final-member content. Every native stdout and exit code is checked against independent fixtures. Explicit `--archive=all --archive-depth=2` enables nested traversal. Native workers are requests, not a claim of parallel decoder work. + +The separate memory matrix has 84 cases x three sessions x four variants: 48 archive cases plus 36 direct-source cases. Source cases cover incapable legacy, custom caller-owned and actual memory sources, range versus whole-result reads, and 0/10/4,096/65,536/1,048,576/16,777,216 bytes. Fixtures and content expectations are outside timed loops. Archive registration uses 0.02 s timing/0.01 s warmup; source registration uses 0.05 s/0.05 s. Every case retains nine repetitions, fastest seven. Batches amortize untimed CPU metadata startup without pooling different cases. Linux uses tmpfs and CPU affinity; macOS uses warm APFS without pinning. + +Confirmation adds ten native cases x five sessions and twelve memory cases x three sessions with a byte-identical baseline participant. Three tiny source cases additionally use one million iterations per repetition, preserving nine repetitions/fastest seven. These fixed-iteration populations remain separate. No same-host timing overlaps builds, lint or instrumentation. Reference-repeat movement is shown, never subtracted. + +## Native commands + +Median paired-session elapsed-time changes; negative means faster. + +| Fixture / action | CPUs | Linux initial | Linux confirm | Linux repeat | macOS initial | macOS confirm | macOS repeat | +| :--------------------------- | ---: | ------------: | ------------: | -----------: | ------------: | ------------: | -----------: | +| ordinary / list | 1 | +1.17% | +1.13% | +0.20% | +0.31% | +0.73% | -0.09% | +| tar-in-tar-10000-4096 / list | 1 | +0.04% | +0.88% | -0.15% | +0.09% | +0.04% | +0.35% | +| tar-10000-4096 / list | 1 | +0.48% | +1.73% | +0.25% | +0.59% | +0.32% | -0.06% | +| tar-in-tgz-10000-4096 / last | 3 | +0.02% | +0.15% | +0.02% | +0.03% | +0.17% | +0.05% | +| tar-in-tar-10000-128 / list | 1 | -0.16% | +2.04% | +1.14% | +0.96% | -0.01% | +0.15% | +| zip-in-tar-10000-4096 / last | 1 | +0.95% | +1.42% | +0.22% | -0.40% | -0.01% | +0.00% | +| zip-10-128 / last | 3 | -1.24% | -0.02% | +0.28% | -0.31% | +1.18% | +0.19% | +| tar-in-tar-10-128 / list | 1 | +0.17% | +1.00% | +0.41% | +1.76% | +0.80% | -0.62% | +| ordinary / last | 3 | -0.44% | +3.68% | +1.91% | -0.63% | +1.05% | +0.11% | +| tar-in-tar-10000-4096 / last | 3 | +0.36% | +1.52% | +0.02% | +0.59% | +0.84% | +0.25% | + +## Direct source kernels + +All 36 source cases are retained. Each cell is the median paired-session fastest-seven elapsed-time change. + +| Source / operation / bytes | Linux range | Linux bytes | Linux combined | macOS range | macOS bytes | macOS combined | +| :-------------------------- | ----------: | ----------: | -------------: | ----------: | ----------: | -------------: | +| caller-owned/bytes/0 | +0.08% | -31.49% | -31.27% | +0.95% | -0.39% | +1.02% | +| caller-owned/bytes/10 | +0.36% | -53.71% | -51.30% | -0.71% | -8.11% | -7.37% | +| caller-owned/bytes/1048576 | +0.47% | -20.82% | -19.80% | -1.41% | -22.00% | -20.57% | +| caller-owned/bytes/16777216 | -0.65% | -16.46% | -16.12% | +0.62% | -18.67% | -20.19% | +| caller-owned/bytes/4096 | -3.15% | -23.42% | -22.31% | -0.32% | -26.12% | -26.20% | +| caller-owned/bytes/65536 | -0.06% | -35.47% | -34.11% | +0.69% | -26.30% | -26.41% | +| caller-owned/range/0 | -0.18% | +0.32% | +0.75% | +3.21% | +4.46% | -1.69% | +| caller-owned/range/10 | -16.55% | -1.65% | -13.66% | -5.75% | +0.44% | -2.59% | +| caller-owned/range/1048576 | -27.74% | -0.96% | -27.59% | -28.01% | +11.84% | -27.84% | +| caller-owned/range/16777216 | -24.75% | +0.66% | -26.24% | -29.20% | -1.08% | -29.21% | +| caller-owned/range/4096 | -28.37% | -0.48% | -27.57% | -28.33% | +0.56% | -27.64% | +| caller-owned/range/65536 | -34.83% | -0.12% | -34.20% | -47.00% | -14.02% | -47.11% | +| legacy/bytes/0 | +0.59% | +6.90% | +4.50% | +0.20% | +10.21% | +16.99% | +| legacy/bytes/10 | -0.75% | -18.44% | -12.87% | +0.98% | +6.93% | +10.57% | +| legacy/bytes/1048576 | +3.68% | +0.35% | +2.85% | +0.47% | -5.21% | -5.02% | +| legacy/bytes/16777216 | +0.10% | +0.91% | -0.53% | -1.64% | -0.82% | -2.05% | +| legacy/bytes/4096 | -0.33% | -30.83% | -32.54% | -2.47% | -28.36% | -29.38% | +| legacy/bytes/65536 | +0.39% | -32.39% | -32.46% | -0.28% | -26.01% | -26.19% | +| legacy/range/0 | -0.12% | +0.37% | +0.01% | -2.36% | -0.63% | -1.26% | +| legacy/range/10 | +1.73% | -0.23% | +6.86% | +0.93% | -0.20% | +2.63% | +| legacy/range/1048576 | -0.87% | -1.10% | -0.01% | +0.08% | -0.06% | -0.49% | +| legacy/range/16777216 | -0.64% | -0.61% | -1.38% | -0.26% | +0.37% | -0.67% | +| legacy/range/4096 | +1.14% | +0.47% | -0.23% | +0.64% | -0.28% | +0.08% | +| legacy/range/65536 | -0.44% | -0.05% | -0.23% | -16.82% | -21.39% | -13.43% | +| memory/bytes/0 | +0.23% | -34.74% | -30.50% | +3.24% | -2.15% | +3.27% | +| memory/bytes/10 | +0.07% | -52.60% | -51.04% | -0.87% | -7.67% | -4.46% | +| memory/bytes/1048576 | -1.86% | -18.83% | -18.22% | -9.22% | -21.51% | -20.36% | +| memory/bytes/16777216 | -0.39% | -15.95% | -16.36% | -0.05% | -19.76% | -21.36% | +| memory/bytes/4096 | +0.64% | -24.25% | -21.55% | +1.44% | -26.50% | -26.67% | +| memory/bytes/65536 | +0.44% | -30.04% | -29.43% | +0.24% | -26.19% | -26.36% | +| memory/range/0 | -0.08% | -0.24% | +0.41% | +0.32% | +2.56% | +3.27% | +| memory/range/10 | -20.85% | +0.95% | -15.98% | -10.73% | +0.61% | -9.82% | +| memory/range/1048576 | -36.17% | -0.16% | -35.76% | -51.48% | -0.47% | -51.64% | +| memory/range/16777216 | -81.91% | +0.22% | -81.99% | -29.26% | -0.14% | -29.48% | +| memory/range/4096 | -30.78% | -0.33% | -29.76% | -29.60% | -1.40% | -29.90% | +| memory/range/65536 | -34.04% | +0.28% | -33.75% | -83.11% | -60.34% | -83.08% | + +## Archive and small controls + +| Case | Linux initial | macOS initial | +| :------------------------------------ | ------------: | ------------: | +| archive/legacy/list/10/128 | -0.09% | +1.61% | +| archive/legacy/list/10000/4096 | -1.29% | +0.30% | +| archive/legacy/last/10000/4096 | -0.56% | -0.98% | +| archive/caller-owned/list/10/128 | -0.44% | -1.36% | +| archive/caller-owned/list/10000/4096 | -0.65% | -0.65% | +| archive/caller-owned/last/10000/4096 | +1.30% | -1.92% | +| archive/memory/list/10/128 | -0.39% | +0.91% | +| archive/memory/list/10000/4096 | +0.12% | -0.25% | +| archive/memory/last/10000/4096 | -0.76% | +0.06% | +| archive/nested-memory/list/10/128 | -0.57% | -1.39% | +| archive/nested-memory/list/10000/4096 | +0.49% | -2.43% | +| archive/nested-memory/last/10000/4096 | +0.50% | +2.90% | + +## Confirmation + +| Case | Phase | Linux combined | Linux repeat | macOS combined | macOS repeat | +| :------------------------------------ | :------------------ | -------------: | -----------: | -------------: | -----------: | +| source/memory/bytes/10 | confirmation-memory | -50.35% | +2.62% | -6.60% | +1.31% | +| source/memory/range/16777216 | confirmation-memory | -82.32% | -0.54% | -29.06% | -0.00% | +| source/caller-owned/range/16777216 | confirmation-memory | -27.25% | -4.34% | -28.46% | -0.04% | +| source/caller-owned/bytes/4096 | confirmation-memory | -21.25% | -0.67% | -26.49% | +0.62% | +| source/legacy/range/4096 | confirmation-memory | +1.27% | -0.09% | -0.73% | -0.96% | +| source/caller-owned/bytes/10 | confirmation-memory | -50.14% | +0.28% | -4.06% | +2.32% | +| source/legacy/bytes/4096 | confirmation-memory | -28.20% | -0.05% | -29.12% | -0.31% | +| source/legacy/bytes/10 | confirmation-memory | -12.32% | -0.13% | +9.66% | +1.52% | +| source/memory/bytes/16777216 | confirmation-memory | -16.54% | -0.64% | -20.78% | +0.13% | +| archive/nested-memory/list/10000/4096 | confirmation-memory | -1.88% | -0.41% | -0.56% | -0.11% | +| archive/legacy/list/10/128 | confirmation-memory | -2.78% | -2.39% | +0.36% | -1.01% | +| archive/nested-memory/last/10000/4096 | confirmation-memory | -3.61% | -3.12% | -0.59% | -0.22% | +| source/legacy/bytes/10 | fixed-small | -12.17% | +0.15% | +9.27% | -0.30% | +| source/caller-owned/bytes/10 | fixed-small | -51.00% | -0.76% | -9.17% | +0.38% | +| source/memory/bytes/10 | fixed-small | -50.40% | +1.77% | -4.84% | -0.09% | + +Tiny fixed-iteration absolute times, median session fastest-seven means, in nanoseconds: + +| Case | Linux baseline ns | Linux selected ns | macOS baseline ns | macOS selected ns | +| :--------------------------- | ----------------: | ----------------: | ----------------: | ----------------: | +| source/legacy/bytes/10 | 39.3 | 34.4 | 34.9 | 38.2 | +| source/caller-owned/bytes/10 | 39.2 | 19.2 | 35.1 | 32.7 | +| source/memory/bytes/10 | 37.8 | 18.7 | 33.8 | 32.7 | + +## Allocation diagnostics + +Warmed calling-thread ASan hooks count requests and cumulative requested bytes. Fixture construction/reporting are excluded. These are not release timings or peak/RSS savings; string capacity growth remains real. + +| Family / case | Host | Baseline requests | Selected requests | Baseline requested bytes | Selected requested bytes | +| :---------------------------------------------- | :---- | ----------------: | ----------------: | -----------------------: | -----------------------: | +| source/memory/range/16777216/real_time | linux | 3 | 2 | 33,554,496 | 16,777,264 | +| source/memory/range/16777216/real_time | macos | 3 | 2 | 33,554,480 | 16,777,256 | +| source/caller-owned/range/16777216/real_time | linux | 258 | 2 | 33,558,584 | 16,777,272 | +| source/caller-owned/range/16777216/real_time | macos | 258 | 2 | 33,556,528 | 16,777,264 | +| source/legacy/bytes/4096/real_time | linux | 3 | 2 | 8,264 | 4,152 | +| source/legacy/bytes/4096/real_time | macos | 3 | 2 | 8,248 | 4,144 | +| source/memory/bytes/16777216/real_time | linux | 266 | 10 | 50,278,416 | 33,505,280 | +| source/memory/bytes/16777216/real_time | macos | 266 | 10 | 50,272,280 | 33,501,192 | +| archive/nested-memory/last/10000/4096/real_time | linux | 60,085 | 60,083 | 2,789,576 | 2,658,472 | +| archive/nested-memory/last/10000/4096/real_time | macos | 60,085 | 60,083 | 2,789,456 | 2,658,368 | + +## Rejected generalized fallback + +The first implementation copied even incapable owned reads through the span callback and used an SSO-sized initial read for them. Its Linux legacy 4 KiB materialization regressed about 24%. Preserve its complete 300 native rows and 1,008 kernel runs per host separately. The refined branch directly reads/appends owned chunks and takes ownership of the first block. Its three-variant pilot keeps 24 native rows and 126 kernel runs per host; final four-variant qualification is a new population. Do not pool the rejected or pilot samples with final results. + +## Correctness and footprint + +Nine committed new cases cover mixed capability/short reads, early EOF with reservation retention, positive exact/zero-limit probes, impossible counts from both APIs, late/probe errors, maximum arithmetic, and owned-only small/multiple-chunk results. Existing archive tests retain sparse PAX/gzip content, parent lifetimes, errors and budgets. The independent format probe now performs zero owned reads instead of one; only that expected counter changes. + +Both hosts pass all 313 selected root-and-extension release suites and five focused ASan suites. Linux passes five MSan and five TSan suites with FUSE required for the complete release selection. Actual scoped clang-tidy passed all four changed translation units per host; the runtime refinement stays within the existing complexity limit. A final test-only assertion covers owned early EOF below the caller limit; its target is rerun under every validated configuration, with production and benchmark inputs unchanged. Seven-file source guards verify exact restoration before/after builds, validation and diagnostics. Normal hooks and this PR's current-head CI remain merge requirements. + +- linux: unstripped full CLI 18,148,600 -> 18,159,248 bytes (+10,648). +- macos: unstripped full CLI 8,158,200 -> 8,158,392 bytes (+192). + +O19b seeking/index/replay/admission remains open; no constant-time archive lookup or newly parallel decoder is claimed. + +## Lossless evidence + +[direct-source-materialization-20261103.json.gz](direct-source-materialization-20261103.json.gz): 5,695,819 bytes; SHA-256 `63160535a98a05869eeff1d2ba357a9f99a62b81ef424c72c7ab4ed92f9d41e3`. diff --git a/docs/performance-analysis.md b/docs/performance-analysis.md index 27ae1e1e55d..fb2e40c6778 100644 --- a/docs/performance-analysis.md +++ b/docs/performance-analysis.md @@ -3584,3 +3584,41 @@ all selected root/extension release tests and focused sanitizers remain part of The initial nested fixture command missed `--archive-depth=2`; its warmup oracle failed before accepting any native row. The corrected matrix uses explicit depth two. O19b seeking/index/replay and admission, and O19d direct final-buffer materialization, remain open. + +### O19d: fill final range and materialization storage + +`ReadSourceRange` already reserves its bounded result, but previously allocated owned intermediate +blocks and copied them into that result. The selected path fills uninitialized final storage and +commits only initialized bytes. It retains the exact range reservation, offset-overflow/64 MiB +checks, zero-range behavior and sequential `OpenAt` replay accounting. Positive short reads +continue; failures never become EOF. + +`ReadSourceBytes` uses the same checked span path when supported, reusing spare capacity and SSO. +Incapable decoders keep direct owned reads and append; the first owned block transfers into the +empty result, avoiding another allocation/copy. Observe capability once for each read, rather than +caching it across cursor progress. At an exact limit, including zero, request one positive byte to +distinguish EOF from overflow. Clamp logical length to the string maximum before arithmetic and +reject impossible counts before committing string size. String capacity can still grow beyond +logical size; this is not an RSS or total-process-memory limit. + +[The cross-platform qualification](benchmarks/direct-source-materialization-20261103.md) separates +range-only, bytes-only and combined variants, direct-source and archive kernels, native commands, +identical-baseline confirmation, fixed-iteration tiny cases, allocation diagnostics and footprint. +Actual memory sources and explicitly incapable legacy sources are distinct. Caller-owned storage +removes copies and intermediate allocations; virtual dispatch, cursor creation, archive indexing +and decoder work remain. An SSO-sized first read may add a backend read for a larger source; its +cost is included rather than assumed free. + +The first generalized span fallback regressed a legacy 4 KiB materialization by about 24% on Linux. +It is rejected. Its complete source/data populations and the separate three-variant refinement +pilot remain in the raw archive. The selected direct owned path and first-block transfer remove +that regression. Keep slower tiny/native controls, their absolute times and reference-repeat +movement visible; do not pool populations or infer universal speedups from favorable large cells. + +Nine committed cases cover changing capability, partial/early EOF, range reservation lifetime, +exact/zero-limit positive probes, malformed counts from both read APIs, late/probe errors, maximum +arithmetic and owned-only small/multiple-chunk results. The archive format probe now also fills +reserved storage directly; its expected owned-read counter changes from one to zero. Sparse holes, +parent lifetimes, content and shared-budget tests remain. No new public flag, external dependency, +worker, lock, metadata probe or host-I/O bypass is introduced. O19b seeking/index/replay/admission +remains open. diff --git a/extra_modules/archive/archive_source_buffer_test.cc b/extra_modules/archive/archive_source_buffer_test.cc index 074325fec43..d2e8e5e5224 100644 --- a/extra_modules/archive/archive_source_buffer_test.cc +++ b/extra_modules/archive/archive_source_buffer_test.cc @@ -169,7 +169,7 @@ TEST_F(ArchiveSourceBufferTest, ReusesTheCallerOwnedBufferForFullAndShortBlocks) const auto source = std::make_shared(MakeTar(std::string(3 * kReadBlock, 'x')), ReadOptions{}, counts); EXPECT_THAT(ListMembersOfSource(source), IsOkAndHolds(ElementsAre(Field(&Member::path, EqualsText("data.txt"))))); - EXPECT_THAT(counts->owned, Eq(1)); // Only the independent four-byte format probe owns its result. + EXPECT_THAT(counts->owned, Eq(0)); // The independent format probe also fills its reserved storage directly. EXPECT_THAT(counts->caller_owned, Gt(3)); EXPECT_THAT(counts->stable_storage, IsTrue()); } diff --git a/release-site.json b/release-site.json index 2eb9d52ff97..2bd84c56fd5 100644 --- a/release-site.json +++ b/release-site.json @@ -79,7 +79,8 @@ "docs/benchmarks/byte-shingle-representations-20261102.md": "docs/benchmarks/byte-shingle-representations-20261102.html", "docs/benchmarks/archive-input-buffers-20261102.md": "docs/benchmarks/archive-input-buffers-20261102.html", "docs/benchmarks/digest-and-record-storage-20261103.md": "docs/benchmarks/digest-and-record-storage-20261103.html", - "docs/benchmarks/caller-owned-streams-20261103.md": "docs/benchmarks/caller-owned-streams-20261103.html" + "docs/benchmarks/caller-owned-streams-20261103.md": "docs/benchmarks/caller-owned-streams-20261103.html", + "docs/benchmarks/direct-source-materialization-20261103.md": "docs/benchmarks/direct-source-materialization-20261103.html" }, "generated_html": { "XFF.md": { diff --git a/tools/BUILD.bazel b/tools/BUILD.bazel index aafdf434aa4..ea2f576880a 100644 --- a/tools/BUILD.bazel +++ b/tools/BUILD.bazel @@ -41,6 +41,21 @@ cc_binary( ], ) +cc_binary( + name = "read_source_benchmark", + srcs = ["read_source_benchmark.cc"], + tags = [ + "clang-tidy", + "manual", + ], + deps = [ + "@abseil-cpp//absl/status:statusor", + "@abseil-cpp//absl/strings", + "@com_github_google_benchmark//:benchmark", + "@xff_extras_api//:read_source_cc", + ], +) + py_library( name = "release_backfill_lib", srcs = ["release_backfill.py"], diff --git a/tools/read_source_benchmark.cc b/tools/read_source_benchmark.cc new file mode 100644 index 00000000000..ba5dc3b32cf --- /dev/null +++ b/tools/read_source_benchmark.cc @@ -0,0 +1,139 @@ +// SPDX-FileCopyrightText: Copyright (c) M. Boerger, the MBO Works authors +// SPDX-License-Identifier: Apache-2.0 + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "absl/status/statusor.h" +#include "absl/strings/str_cat.h" +#include "benchmark/benchmark.h" +#include "xff/vfs/read_source.h" + +namespace xff::vfs { +namespace { +constexpr std::size_t kChunk = 64UZ * 1'024; + +class BenchmarkStream final : public ReadStream { + public: + BenchmarkStream(std::shared_ptr bytes, bool caller_owned) + : bytes_(std::move(bytes)), caller_owned_(caller_owned) {} + + absl::StatusOr Read(std::size_t max_bytes) override { + const auto count = std::min({max_bytes, kChunk, bytes_->size() - offset_}); + auto result = bytes_->substr(offset_, count); + offset_ += count; + return result; + } + + bool SupportsReadInto() const noexcept override { return caller_owned_; } + + absl::StatusOr ReadInto(std::span buffer) override { + const auto count = std::min({buffer.size(), kChunk, bytes_->size() - offset_}); + std::ranges::copy(std::string_view(*bytes_).substr(offset_, count), buffer.begin()); + offset_ += count; + return count; + } + + private: + std::shared_ptr bytes_; + bool caller_owned_; + std::size_t offset_ = 0; +}; + +class BenchmarkSource final : public ReadSource { + public: + BenchmarkSource(std::string bytes, bool caller_owned) + : bytes_(std::make_shared(std::move(bytes))), caller_owned_(caller_owned) {} + + absl::StatusOr> Open() const override { + return std::make_unique(bytes_, caller_owned_); + } + + private: + std::shared_ptr bytes_; + bool caller_owned_; +}; + +enum class SourceKind { kLegacy, kCallerOwned, kMemory }; + +SharedReadSource MakeSource(SourceKind kind, std::string bytes) { + if (kind == SourceKind::kMemory) { + return MemoryReadSource(std::move(bytes)); + } + return std::make_shared(std::move(bytes), kind == SourceKind::kCallerOwned); +} + +void Measure(benchmark::State& state, SourceKind kind, bool range) { + const auto count = static_cast(state.range(0)); + std::string expected(count, 'x'); + if (!expected.empty()) { + expected.front() = 'a'; + expected.back() = 'z'; + } + // Sequential custom sources replay this small prefix; memory sources seek directly. + const auto source = MakeSource(kind, range ? absl::StrCat("prefix:", expected, ":suffix") : expected); + // NOLINTNEXTLINE(clang-analyzer-deadcode.DeadStores): benchmark controls the measured iterations. + for (const auto unused : state) { + if (range) { + auto result = ReadSourceRange(*source, 7, count); + if (!result.ok() || result->bytes != expected) { + state.SkipWithError("range differs from fixture"); + return; + } + benchmark::DoNotOptimize(result); + } else { + auto result = ReadSourceBytes(*source, count); + if (!result.ok() || *result != expected) { + state.SkipWithError("materialization differs from fixture"); + return; + } + benchmark::DoNotOptimize(result); + } + } + state.SetBytesProcessed(state.iterations() * static_cast(count)); +} + +void RegisterAll() { + struct SourceCase { + std::string_view name; + SourceKind kind; + }; + + constexpr auto kSources = std::to_array({ + {.name = "legacy", .kind = SourceKind::kLegacy}, + {.name = "caller-owned", .kind = SourceKind::kCallerOwned}, + {.name = "memory", .kind = SourceKind::kMemory}, + }); + constexpr auto kCounts = std::to_array({0, 10, 4'096, 65'536, 1'048'576, 16'777'216}); + for (const auto& source : kSources) { + for (const bool range : {false, true}) { + const std::string name = absl::StrCat("source/", source.name, "/", range ? "range" : "bytes"); + // XFF_ABI_POINTER: Google Benchmark returns a borrowed mutable registration object. + auto* registration = benchmark::RegisterBenchmark(name, Measure, source.kind, range); + for (const auto count : kCounts) { + registration->Arg(count); + } + registration->Unit(benchmark::kMicrosecond)->UseRealTime()->MinTime(0.05)->MinWarmUpTime(0.05)->Repetitions(9); + } + } +} +} // namespace +} // namespace xff::vfs + +// XFF_ABI_POINTER: Google Benchmark's process argument interface. +int main(int argc, char** argv) { + benchmark::Initialize(&argc, argv); + xff::vfs::RegisterAll(); + if (benchmark::ReportUnrecognizedArguments(argc, argv)) { + return 1; + } + benchmark::RunSpecifiedBenchmarks(); + benchmark::Shutdown(); +} diff --git a/xff_extras_api/BUILD.bazel b/xff_extras_api/BUILD.bazel index 09eb9e25a55..0e5471f10f7 100644 --- a/xff_extras_api/BUILD.bazel +++ b/xff_extras_api/BUILD.bazel @@ -404,6 +404,7 @@ cc_test( ":vfs_cc", "@abseil-cpp//absl/status", "@googletest//:gtest_main", + "@mboworks_mbo//mbo/status:status_macros_cc", "@mboworks_mbo//mbo/testing:matchers_cc", "@mboworks_mbo//mbo/testing:status_cc", ], diff --git a/xff_extras_api/read_source.cc b/xff_extras_api/read_source.cc index 843c4cfa0a4..75da835353a 100644 --- a/xff_extras_api/read_source.cc +++ b/xff_extras_api/read_source.cc @@ -8,6 +8,7 @@ #include #include +#include #include #include #include @@ -21,6 +22,35 @@ namespace xff::vfs { namespace { +// Keep capability changes and owned-only decoders valid for every individual read. +absl::StatusOr ReadStreamInto(ReadStream& stream, std::span buffer, bool caller_owned) { + if (caller_owned) { + MBO_ASSIGN_OR_RETURN(const auto count, stream.ReadInto(buffer)); + if (count > buffer.size()) { + return absl::DataLossError("read source returned more bytes than requested"); + } + return count; + } + MBO_ASSIGN_OR_RETURN(const auto bytes, stream.Read(buffer.size())); + if (bytes.size() > buffer.size()) { + return absl::DataLossError("read source returned more bytes than requested"); + } + std::ranges::copy(bytes, buffer.begin()); + return bytes.size(); +} + +absl::StatusOr ReadStreamFullyInto(ReadStream& stream, std::span buffer) { + std::size_t filled = 0; + while (filled < buffer.size()) { + MBO_ASSIGN_OR_RETURN(const auto count, ReadStreamInto(stream, buffer.subspan(filled), stream.SupportsReadInto())); + if (count == 0) { + break; + } + filled += count; + } + return filled; +} + class MemoryStream final : public ReadStream { public: MemoryStream(std::shared_ptr bytes, std::size_t offset) @@ -192,29 +222,65 @@ absl::StatusOr ReadSourceRange(const ReadSource& source, std::uint64_ return result; } MBO_ASSIGN_OR_RETURN(auto stream, source.OpenAt(offset)); - result.bytes.reserve(count); - while (result.bytes.size() < count) { - MBO_ASSIGN_OR_RETURN(const auto bytes, stream->Read(count - result.bytes.size())); - if (bytes.empty()) { - break; - } - result.bytes += bytes; - } + absl::StatusOr transferred = 0; + // XFF_ABI_POINTER: std::string::resize_and_overwrite requires this writable buffer callback. + result.bytes.resize_and_overwrite(count, [&](char* bytes, std::size_t) noexcept -> std::size_t { + transferred = ReadStreamFullyInto(*stream, std::span(bytes, count)); + // Callback errors remain in transferred; no uninitialized bytes become string content. + return transferred.value_or(0); + }); + MBO_RETURN_IF_ERROR(transferred.status()); return result; } absl::StatusOr ReadSourceBytes(const ReadSource& source, std::size_t max_bytes) { MBO_ASSIGN_OR_RETURN(auto stream, source.Open()); std::string result; + const std::size_t limit = std::min(max_bytes, result.max_size()); while (true) { - MBO_ASSIGN_OR_RETURN(const std::string block, stream->Read(64UZ * 1'024)); - if (block.empty()) { - return result; - } - if (block.size() > max_bytes - result.size()) { + if (result.size() == limit) { + // A positive-sized read distinguishes exact EOF from overflow, including a zero limit. + std::array probe; + MBO_ASSIGN_OR_RETURN(const auto count, ReadStreamInto(*stream, probe, stream->SupportsReadInto())); + if (count == 0) { + return result; + } return absl::ResourceExhaustedError("container exceeds materialization limit"); } - result += block; + const std::size_t previous = result.size(); + if (!stream->SupportsReadInto()) { + const std::size_t request = std::min(64UZ * 1'024, limit - previous); + MBO_ASSIGN_OR_RETURN(auto bytes, stream->Read(request)); + if (bytes.size() > request) { + return absl::DataLossError("read source returned more bytes than requested"); + } + if (bytes.empty()) { + return result; + } + if (previous == 0) { + result = std::move(bytes); + continue; + } + result += bytes; + continue; + } + std::size_t request = std::min(64UZ * 1'024, limit - previous); + if (previous < result.capacity()) { + // Reuse spare storage after a short read, including SSO; probing EOF must not grow it. + request = std::min(request, result.capacity() - previous); + } + absl::StatusOr transferred = 0; + // XFF_ABI_POINTER: std::string::resize_and_overwrite requires this writable buffer callback. + result.resize_and_overwrite(previous + request, [&](char* bytes, std::size_t) noexcept -> std::size_t { + transferred = ReadStreamInto(*stream, std::span(bytes, previous + request).subspan(previous), true); + // Preserve errors separately because this standard callback returns only the initialized size. + return previous + transferred.value_or(0); + }); + MBO_ASSIGN_OR_RETURN(const auto count, std::move(transferred)); + if (count == 0) { + return result; + } } } + } // namespace xff::vfs diff --git a/xff_extras_api/read_source_test.cc b/xff_extras_api/read_source_test.cc index 16928199896..6fd818c6ccc 100644 --- a/xff_extras_api/read_source_test.cc +++ b/xff_extras_api/read_source_test.cc @@ -3,6 +3,8 @@ #include "xff/vfs/read_source.h" +#include +#include #include #include #include @@ -13,6 +15,7 @@ #include "absl/status/status.h" #include "gmock/gmock.h" #include "gtest/gtest.h" +#include "mbo/status/status_macros.h" #include "mbo/testing/matchers.h" #include "mbo/testing/status.h" #include "xff/vfs/mutations.h" @@ -23,8 +26,10 @@ using ::mbo::testing::EqualsText; using ::mbo::testing::IsOk; using ::mbo::testing::IsOkAndHolds; using ::mbo::testing::StatusIs; +using ::testing::ElementsAre; using ::testing::Eq; using ::testing::Field; +using ::testing::FieldsAre; using ::testing::IsEmpty; using ::testing::IsFalse; using ::testing::IsTrue; @@ -267,5 +272,191 @@ TEST_F(ReadSourceTest, HostCursorExposesOnlyInitializedBytesAndRetainsEarlierChu EXPECT_THAT(unreadable->Read(1), StatusIs(absl::StatusCode::kDataLoss)); } +enum class ProbeReadKind { kOwned, kInto }; + +struct ProbeReadRequest { + ProbeReadKind kind; + std::size_t count; +}; + +struct ProbeReadState { + std::string bytes = "abcdefg"; + std::vector kinds = {ProbeReadKind::kInto, ProbeReadKind::kOwned}; + std::size_t chunk = 2; + std::size_t fail_call = std::numeric_limits::max(); + bool oversized = false; + std::vector requests; +}; + +class ProbeReadStream final : public ReadStream { + public: + explicit ProbeReadStream(std::shared_ptr state) : state_(std::move(state)) {} + + bool SupportsReadInto() const noexcept override { + return state_->kinds.at(state_->requests.size() % state_->kinds.size()) == ProbeReadKind::kInto; + } + + absl::StatusOr Read(std::size_t max_bytes) override { + MBO_ASSIGN_OR_RETURN(const auto count, Count(max_bytes, ProbeReadKind::kOwned)); + if (state_->oversized) { + return std::string(max_bytes + 1, '!'); + } + auto bytes = state_->bytes.substr(offset_, count); + offset_ += count; + return bytes; + } + + absl::StatusOr ReadInto(std::span buffer) override { + MBO_ASSIGN_OR_RETURN(const auto count, Count(buffer.size(), ProbeReadKind::kInto)); + if (state_->oversized) { + return buffer.size() + 1; + } + std::ranges::copy(std::string_view(state_->bytes).substr(offset_, count), buffer.begin()); + offset_ += count; + return count; + } + + private: + absl::StatusOr Count(std::size_t requested, ProbeReadKind kind) { + const auto call = state_->requests.size(); + state_->requests.push_back({.kind = kind, .count = requested}); + if (call == state_->fail_call) { + return absl::DataLossError("injected source failure"); + } + return std::min({requested, state_->chunk, state_->bytes.size() - offset_}); + } + + std::shared_ptr state_; + std::size_t offset_ = 0; +}; + +class ProbeReadSource final : public ReadSource { + public: + explicit ProbeReadSource(std::shared_ptr state) + : ReadSource(std::make_shared(128, 128)), state_(std::move(state)) {} + + absl::StatusOr> Open() const override { + return std::make_unique(state_); + } + + private: + std::shared_ptr state_; +}; + +TEST_F(ReadSourceTest, DirectRangePreservesShortReadsAndChangingCapabilities) { + const auto state = std::make_shared(); + state->bytes = "abcdefgh"; + const ProbeReadSource source(state); + { + MBO_ASSERT_OK_AND_ASSIGN(const auto block, ReadSourceRange(source, 0, 8)); + EXPECT_THAT(block.bytes, EqualsText("abcdefgh")); + EXPECT_THAT(source.Budget()->MemoryUsed(), Eq(8)); + EXPECT_THAT( + state->requests, ElementsAre( + FieldsAre(Eq(ProbeReadKind::kInto), Eq(8)), FieldsAre(Eq(ProbeReadKind::kOwned), Eq(6)), + FieldsAre(Eq(ProbeReadKind::kInto), Eq(4)), FieldsAre(Eq(ProbeReadKind::kOwned), Eq(2)))); + } + EXPECT_THAT(source.Budget()->MemoryUsed(), Eq(0)); +} + +TEST_F(ReadSourceTest, DirectRangeRetainsReservationAndOnlyInitializedPrefixAtEarlyEof) { + const auto state = std::make_shared(); + state->bytes = "abc"; + const ProbeReadSource source(state); + MBO_ASSERT_OK_AND_ASSIGN(const auto block, ReadSourceRange(source, 0, 10)); + EXPECT_THAT(block.bytes, EqualsText("abc")); + EXPECT_THAT(source.Budget()->MemoryUsed(), Eq(10)); + EXPECT_THAT( + state->requests, ElementsAre( + FieldsAre(Eq(ProbeReadKind::kInto), Eq(10)), FieldsAre(Eq(ProbeReadKind::kOwned), Eq(8)), + FieldsAre(Eq(ProbeReadKind::kInto), Eq(7)))); +} + +TEST_F(ReadSourceTest, MaterializationUsesPositiveProbeAtExactLimitWithChangingCapabilities) { + const auto state = std::make_shared(); + EXPECT_THAT(ReadSourceBytes(ProbeReadSource(state), 7), IsOkAndHolds(EqualsText("abcdefg"))); + EXPECT_THAT( + state->requests, ElementsAre( + FieldsAre(Eq(ProbeReadKind::kInto), Eq(7)), FieldsAre(Eq(ProbeReadKind::kOwned), Eq(5)), + FieldsAre(Eq(ProbeReadKind::kInto), Eq(3)), FieldsAre(Eq(ProbeReadKind::kOwned), Eq(1)), + FieldsAre(Eq(ProbeReadKind::kInto), Eq(1)))); +} + +TEST_F(ReadSourceTest, ZeroMaterializationLimitProbesOneByteForEitherCapability) { + constexpr auto kKinds = std::to_array({ProbeReadKind::kOwned, ProbeReadKind::kInto}); + for (const auto kind : kKinds) { + const auto state = std::make_shared(); + state->kinds = {kind}; + EXPECT_THAT(ReadSourceBytes(ProbeReadSource(state), 0), StatusIs(absl::StatusCode::kResourceExhausted)); + EXPECT_THAT(state->requests, ElementsAre(FieldsAre(Eq(kind), Eq(1)))); + state->bytes.clear(); + state->requests.clear(); + EXPECT_THAT(ReadSourceBytes(ProbeReadSource(state), 0), IsOkAndHolds(IsEmpty())); + EXPECT_THAT(state->requests, ElementsAre(FieldsAre(Eq(kind), Eq(1)))); + } +} + +TEST_F(ReadSourceTest, OversizedOwnedAndBorrowedReadsFailBeforeExposingData) { + constexpr auto kKinds = std::to_array({ProbeReadKind::kOwned, ProbeReadKind::kInto}); + for (const auto kind : kKinds) { + const auto state = std::make_shared(); + state->kinds = {kind}; + state->oversized = true; + const ProbeReadSource source(state); + EXPECT_THAT(ReadSourceRange(source, 0, 4), StatusIs(absl::StatusCode::kDataLoss)); + EXPECT_THAT(source.Budget()->MemoryUsed(), Eq(0)); + EXPECT_THAT(ReadSourceBytes(source, 4), StatusIs(absl::StatusCode::kDataLoss)); + EXPECT_THAT(ReadSourceBytes(source, 0), StatusIs(absl::StatusCode::kDataLoss)); + } +} + +TEST_F(ReadSourceTest, LateReadAndLimitProbeErrorsStayErrorsAndReleaseRangeReservation) { + constexpr auto kCalls = std::to_array({0, 1}); + for (const auto call : kCalls) { + const auto state = std::make_shared(); + state->fail_call = call; + const ProbeReadSource source(state); + EXPECT_THAT(ReadSourceRange(source, 0, 4), StatusIs(absl::StatusCode::kDataLoss)); + EXPECT_THAT(source.Budget()->MemoryUsed(), Eq(0)); + state->requests.clear(); + EXPECT_THAT(ReadSourceBytes(source, 4), StatusIs(absl::StatusCode::kDataLoss)); + } + const auto state = std::make_shared(); + state->bytes = "ab"; + state->fail_call = 1; + EXPECT_THAT(ReadSourceBytes(ProbeReadSource(state), 2), StatusIs(absl::StatusCode::kDataLoss)); + EXPECT_THAT( + state->requests, + ElementsAre(FieldsAre(Eq(ProbeReadKind::kInto), Eq(2)), FieldsAre(Eq(ProbeReadKind::kOwned), Eq(1)))); +} + +TEST_F(ReadSourceTest, MaximumMaterializationLimitDoesNotOverflowRequestArithmetic) { + const auto state = std::make_shared(); + state->bytes = "abc"; + EXPECT_THAT( + ReadSourceBytes(ProbeReadSource(state), std::numeric_limits::max()), + IsOkAndHolds(EqualsText("abc"))); +} + +TEST_F(ReadSourceTest, OwnedOnlyMaterializationPreservesBothSmallAndMultiChunkResults) { + constexpr auto kSizes = std::to_array({10, 512}); + for (const auto size : kSizes) { + const auto state = std::make_shared(); + state->bytes = std::string(size, 'x'); + state->bytes.front() = 'a'; + state->bytes.back() = 'z'; + state->chunk = 31; + state->kinds = {ProbeReadKind::kOwned}; + const ProbeReadSource source(state); + EXPECT_THAT(ReadSourceBytes(source, size), IsOkAndHolds(EqualsText(state->bytes))); + state->requests.clear(); + EXPECT_THAT(ReadSourceBytes(source, size + 7), IsOkAndHolds(EqualsText(state->bytes))); + state->requests.clear(); + state->fail_call = 1; + state->chunk = 2; + EXPECT_THAT(ReadSourceBytes(source, size), StatusIs(absl::StatusCode::kDataLoss)); + } +} + } // namespace } // namespace xff::vfs