From f540887176edec861c70eff30b8cc652bacab355 Mon Sep 17 00:00:00 2001 From: Vinit Kumar Date: Wed, 12 Aug 2026 23:24:31 +0530 Subject: [PATCH 1/5] docs: record security benchmark tradeoffs --- BENCHMARKS.md | 40 ++++++++++++++++++++++++ docs/benchmarks.rst | 70 ++++++++++++++++++++++++++++++++++++++++++ lat.md/architecture.md | 4 +++ 3 files changed, 114 insertions(+) diff --git a/BENCHMARKS.md b/BENCHMARKS.md index f2ba1a28..e4c44531 100644 --- a/BENCHMARKS.md +++ b/BENCHMARKS.md @@ -91,6 +91,46 @@ This rerun compares the same CLI workload across uv-managed CPython 3.14.6, CPyt - **Go remained the fastest CLI path overall**, mainly because the conversion work dominates process startup once the payload gets large. - These numbers are **end-to-end subprocess timings**, not isolated serializer throughput, so interpreter startup and environment activation costs are part of the result by design. +### Security Hardening Benchmark (August 12, 2026) + +This comparison measures the public `Json2xml(...).to_xml()` path before and after conversion limits, lexical pretty printing, and compact output by default were added. + +It compares pre-hardening commit `826439f` with hardened `master` at `48dfd38`. The direct `dicttoxml` serializer did not change between these revisions, so `benchmark_all.py` would not expose the wrapper cost. + +#### Method + +- Apple Silicon, macOS 26.6.1, CPython 3.14.6. +- Deterministic small, 100-record, and 1,000-record nested payloads. +- Default, explicit `pretty=False`, and explicit `pretty=True` calls. +- Five warmups per worker and 68 timed samples per cell across four fresh workers per revision. +- Revisions were interleaved in ABBA order to reduce thermal and scheduler bias. + +#### CPython Results + +Each timing is the median public-API conversion time. “Faster” and “slower” compare hardened `master` with the pre-hardening revision. + +| Workload | Mode | Pre-hardening | Hardened master | Change | +|----------|------|--------------:|----------------:|-------:| +| Small | Default | 24.1µs | 6.3µs | **74.0% faster** | +| Small | Compact | 4.4µs | 6.4µs | **44.9% slower** | +| Small | Pretty | 24.3µs | 15.1µs | **38.0% faster** | +| 100 records | Default | 7.07ms | 1.80ms | **74.6% faster** | +| 100 records | Compact | 1.14ms | 1.83ms | **60.6% slower** | +| 100 records | Pretty | 6.87ms | 5.04ms | **26.7% faster** | +| 1,000 records | Default | 86.79ms | 17.44ms | **79.9% faster** | +| 1,000 records | Compact | 10.92ms | 17.40ms | **59.3% slower** | +| 1,000 records | Pretty | 85.03ms | 49.26ms | **42.1% faster** | + +#### Interpretation + +Default calls are roughly 4-5x faster because they now return compact serializer bytes instead of building pretty output. This comparison includes the intentional default-output contract change. + +Explicit pretty output is 27-42% faster because bounded lexical indentation replaces DOM parsing. Its formatting and encoded size differ from the old `minidom` output. + +Explicit compact output is 45-61% slower. Compact bytes were identical across revisions, and the serializer was unchanged, isolating the regression mainly to the new full-input resource-budget scan. + +PyPy 3.10.16 corroborated the tradeoff: default calls were 73-87% faster, pretty calls were 42-72% faster, and compact calls were 31-35% slower. + ## Key Observations ### 1. Rust Extension is the Best Choice for Python Users 🦀 diff --git a/docs/benchmarks.rst b/docs/benchmarks.rst index 2bd175ee..ac442437 100644 --- a/docs/benchmarks.rst +++ b/docs/benchmarks.rst @@ -136,6 +136,76 @@ Speedup vs Pure Python *CLI tools have process spawn overhead of about 3-6ms, which dominates for small inputs.* +Security Hardening Benchmark +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +This August 12, 2026 comparison measures the public ``Json2xml(...).to_xml()`` path before and after conversion limits, lexical pretty printing, and compact output by default were added. + +It compares pre-hardening commit ``826439f`` with hardened ``master`` at ``48dfd38`` on Apple Silicon, macOS 26.6.1, and CPython 3.14.6. + +The deterministic benchmark used small, 100-record, and 1,000-record nested payloads. Each cell reports the median of 68 timed samples across four fresh workers per revision, interleaved in ABBA order after five warmups. + +.. list-table:: + :header-rows: 1 + :widths: 24 18 20 20 18 + + * - Workload + - Mode + - Pre-hardening + - Hardened master + - Change + * - Small + - Default + - 24.1µs + - 6.3µs + - **74.0% faster** + * - Small + - Compact + - 4.4µs + - 6.4µs + - **44.9% slower** + * - Small + - Pretty + - 24.3µs + - 15.1µs + - **38.0% faster** + * - 100 records + - Default + - 7.07ms + - 1.80ms + - **74.6% faster** + * - 100 records + - Compact + - 1.14ms + - 1.83ms + - **60.6% slower** + * - 100 records + - Pretty + - 6.87ms + - 5.04ms + - **26.7% faster** + * - 1,000 records + - Default + - 86.79ms + - 17.44ms + - **79.9% faster** + * - 1,000 records + - Compact + - 10.92ms + - 17.40ms + - **59.3% slower** + * - 1,000 records + - Pretty + - 85.03ms + - 49.26ms + - **42.1% faster** + +Default calls are roughly 4-5x faster because they now return compact serializer bytes. Explicit pretty output is 27-42% faster because lexical indentation replaces DOM parsing. + +Explicit compact output is 45-61% slower. Its bytes were identical and the serializer was unchanged, isolating the regression mainly to the new full-input resource-budget scan. + +PyPy 3.10.16 corroborated the tradeoff: default calls were 73-87% faster, pretty calls were 42-72% faster, and compact calls were 31-35% slower. + Key Observations ---------------- diff --git a/lat.md/architecture.md b/lat.md/architecture.md index e68f6a19..de1c6f2a 100644 --- a/lat.md/architecture.md +++ b/lat.md/architecture.md @@ -58,6 +58,10 @@ The May 2026 benchmark on Apple Silicon shows the Rust extension as the best opt Reproduction docs require contributors to record machine, OS, Python, and tool availability before comparing results. `benchmark_all.py` mixes library calls and CLI subprocesses intentionally, so its Go and Zig rows include process startup overhead. +The August 2026 public-wrapper benchmark compares pre-hardening `826439f` with hardened `48dfd38` using ABBA-interleaved CPython 3.14 samples. + +Default calls improved 74-80% and pretty calls 27-42%, while explicit compact calls regressed 45-61% from the resource-budget scan. + The June 2026 Rust memory benchmark uses [[benchmark_memory_rust.py#main]] under hyperfine to compare release builds in fresh Python processes. The bytes-writer implementation cuts serializer peak RSS by about half for large outputs, with a documented throughput tradeoff. The June 2026 multi-interpreter CLI rerun uses [[benchmark_multi_python.py#main]] with per-interpreter virtual environments. On the recorded Apple Silicon run, CPython 3.15.0rc1 beat CPython 3.14.6 on every case, PyPy 3.11.15 only won the largest case, and Go remained the fastest end-to-end CLI path overall. From 54a9f33eede87858c5192c24e50e1f38d1f610f7 Mon Sep 17 00:00:00 2001 From: Vinit Kumar Date: Wed, 12 Aug 2026 23:29:34 +0530 Subject: [PATCH 2/5] docs: add Python 3.15 security benchmark --- BENCHMARKS.md | 21 +++++++++++++- docs/benchmarks.rst | 64 +++++++++++++++++++++++++++++++++++++++++- lat.md/architecture.md | 2 ++ 3 files changed, 85 insertions(+), 2 deletions(-) diff --git a/BENCHMARKS.md b/BENCHMARKS.md index e4c44531..1f5fcc13 100644 --- a/BENCHMARKS.md +++ b/BENCHMARKS.md @@ -99,7 +99,8 @@ It compares pre-hardening commit `826439f` with hardened `master` at `48dfd38`. #### Method -- Apple Silicon, macOS 26.6.1, CPython 3.14.6. +- Apple Silicon and macOS 26.6.1, with CPython 3.14.6 for the primary run. +- CPython 3.15.0rc1 follow-up downloaded and managed by uv 0.12.3. - Deterministic small, 100-record, and 1,000-record nested payloads. - Default, explicit `pretty=False`, and explicit `pretty=True` calls. - Five warmups per worker and 68 timed samples per cell across four fresh workers per revision. @@ -121,6 +122,22 @@ Each timing is the median public-API conversion time. “Faster” and “slower | 1,000 records | Compact | 10.92ms | 17.40ms | **59.3% slower** | | 1,000 records | Pretty | 85.03ms | 49.26ms | **42.1% faster** | +#### CPython 3.15.0rc1 Results + +The rc1 follow-up uses the same payloads, modes, warmups, sample counts, worker isolation, and ABBA revision ordering as the CPython 3.14.6 run. + +| Workload | Mode | Pre-hardening | Hardened master | Change | +|----------|------|--------------:|----------------:|-------:| +| Small | Default | 24.7µs | 6.2µs | **74.8% faster** | +| Small | Compact | 4.4µs | 6.4µs | **44.0% slower** | +| Small | Pretty | 24.7µs | 14.8µs | **40.0% faster** | +| 100 records | Default | 7.13ms | 1.75ms | **75.5% faster** | +| 100 records | Compact | 1.09ms | 1.75ms | **60.3% slower** | +| 100 records | Pretty | 6.85ms | 4.96ms | **27.5% faster** | +| 1,000 records | Default | 88.01ms | 17.24ms | **80.4% faster** | +| 1,000 records | Compact | 10.90ms | 17.21ms | **57.8% slower** | +| 1,000 records | Pretty | 88.83ms | 50.46ms | **43.2% faster** | + #### Interpretation Default calls are roughly 4-5x faster because they now return compact serializer bytes instead of building pretty output. This comparison includes the intentional default-output contract change. @@ -129,6 +146,8 @@ Explicit pretty output is 27-42% faster because bounded lexical indentation repl Explicit compact output is 45-61% slower. Compact bytes were identical across revisions, and the serializer was unchanged, isolating the regression mainly to the new full-input resource-budget scan. +CPython 3.15.0rc1 reproduces the same tradeoff: default and pretty output improve substantially, while explicit compact conversion pays for the security scan. + PyPy 3.10.16 corroborated the tradeoff: default calls were 73-87% faster, pretty calls were 42-72% faster, and compact calls were 31-35% slower. ## Key Observations diff --git a/docs/benchmarks.rst b/docs/benchmarks.rst index ac442437..7eb927d0 100644 --- a/docs/benchmarks.rst +++ b/docs/benchmarks.rst @@ -141,7 +141,7 @@ Security Hardening Benchmark This August 12, 2026 comparison measures the public ``Json2xml(...).to_xml()`` path before and after conversion limits, lexical pretty printing, and compact output by default were added. -It compares pre-hardening commit ``826439f`` with hardened ``master`` at ``48dfd38`` on Apple Silicon, macOS 26.6.1, and CPython 3.14.6. +It compares pre-hardening commit ``826439f`` with hardened ``master`` at ``48dfd38`` on Apple Silicon and macOS 26.6.1. The primary run used CPython 3.14.6. The deterministic benchmark used small, 100-record, and 1,000-record nested payloads. Each cell reports the median of 68 timed samples across four fresh workers per revision, interleaved in ABBA order after five warmups. @@ -200,10 +200,72 @@ The deterministic benchmark used small, 100-record, and 1,000-record nested payl - 49.26ms - **42.1% faster** +CPython 3.15.0rc1 Follow-up +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +uv 0.12.3 downloaded and managed the exact ``cpython-3.15.0rc1-macos-aarch64-none`` build. The follow-up uses the same payloads, modes, warmups, sample counts, worker isolation, and ABBA ordering. + +.. list-table:: + :header-rows: 1 + :widths: 24 18 20 20 18 + + * - Workload + - Mode + - Pre-hardening + - Hardened master + - Change + * - Small + - Default + - 24.7µs + - 6.2µs + - **74.8% faster** + * - Small + - Compact + - 4.4µs + - 6.4µs + - **44.0% slower** + * - Small + - Pretty + - 24.7µs + - 14.8µs + - **40.0% faster** + * - 100 records + - Default + - 7.13ms + - 1.75ms + - **75.5% faster** + * - 100 records + - Compact + - 1.09ms + - 1.75ms + - **60.3% slower** + * - 100 records + - Pretty + - 6.85ms + - 4.96ms + - **27.5% faster** + * - 1,000 records + - Default + - 88.01ms + - 17.24ms + - **80.4% faster** + * - 1,000 records + - Compact + - 10.90ms + - 17.21ms + - **57.8% slower** + * - 1,000 records + - Pretty + - 88.83ms + - 50.46ms + - **43.2% faster** + Default calls are roughly 4-5x faster because they now return compact serializer bytes. Explicit pretty output is 27-42% faster because lexical indentation replaces DOM parsing. Explicit compact output is 45-61% slower. Its bytes were identical and the serializer was unchanged, isolating the regression mainly to the new full-input resource-budget scan. +CPython 3.15.0rc1 reproduces the same tradeoff: default and pretty output improve substantially, while explicit compact conversion pays for the security scan. + PyPy 3.10.16 corroborated the tradeoff: default calls were 73-87% faster, pretty calls were 42-72% faster, and compact calls were 31-35% slower. Key Observations diff --git a/lat.md/architecture.md b/lat.md/architecture.md index de1c6f2a..79d4fc09 100644 --- a/lat.md/architecture.md +++ b/lat.md/architecture.md @@ -62,6 +62,8 @@ The August 2026 public-wrapper benchmark compares pre-hardening `826439f` with h Default calls improved 74-80% and pretty calls 27-42%, while explicit compact calls regressed 45-61% from the resource-budget scan. +An identical uv-managed CPython 3.15.0rc1 follow-up confirmed the result: default calls improved 75-80%, pretty calls improved 28-43%, and compact calls regressed 44-60%. + The June 2026 Rust memory benchmark uses [[benchmark_memory_rust.py#main]] under hyperfine to compare release builds in fresh Python processes. The bytes-writer implementation cuts serializer peak RSS by about half for large outputs, with a documented throughput tradeoff. The June 2026 multi-interpreter CLI rerun uses [[benchmark_multi_python.py#main]] with per-interpreter virtual environments. On the recorded Apple Silicon run, CPython 3.15.0rc1 beat CPython 3.14.6 on every case, PyPy 3.11.15 only won the largest case, and Go remained the fastest end-to-end CLI path overall. From ae5f9fd331d36a7987536bd862e05f440db01b4b Mon Sep 17 00:00:00 2001 From: Vinit Kumar Date: Wed, 12 Aug 2026 23:31:36 +0530 Subject: [PATCH 3/5] docs: detail Python 3.15 benchmark evidence --- BENCHMARKS.md | 43 +++++++++++++++++------ docs/benchmarks.rst | 79 +++++++++++++++++++++++++++++++----------- lat.md/architecture.md | 2 ++ 3 files changed, 92 insertions(+), 32 deletions(-) diff --git a/BENCHMARKS.md b/BENCHMARKS.md index 1f5fcc13..60e7552b 100644 --- a/BENCHMARKS.md +++ b/BENCHMARKS.md @@ -126,17 +126,38 @@ Each timing is the median public-API conversion time. “Faster” and “slower The rc1 follow-up uses the same payloads, modes, warmups, sample counts, worker isolation, and ABBA revision ordering as the CPython 3.14.6 run. -| Workload | Mode | Pre-hardening | Hardened master | Change | -|----------|------|--------------:|----------------:|-------:| -| Small | Default | 24.7µs | 6.2µs | **74.8% faster** | -| Small | Compact | 4.4µs | 6.4µs | **44.0% slower** | -| Small | Pretty | 24.7µs | 14.8µs | **40.0% faster** | -| 100 records | Default | 7.13ms | 1.75ms | **75.5% faster** | -| 100 records | Compact | 1.09ms | 1.75ms | **60.3% slower** | -| 100 records | Pretty | 6.85ms | 4.96ms | **27.5% faster** | -| 1,000 records | Default | 88.01ms | 17.24ms | **80.4% faster** | -| 1,000 records | Compact | 10.90ms | 17.21ms | **57.8% slower** | -| 1,000 records | Pretty | 88.83ms | 50.46ms | **43.2% faster** | +uv 0.11.24 did not yet list rc1, so it was updated to 0.12.3. That release downloaded the exact 26.2 MiB Apple Silicon build and verified its version before measurement. + +```bash +uv self update +uv python list 3.15 --all-versions +uv run --isolated --managed-python \ + --python cpython-3.15.0rc1-macos-aarch64-none python --version +``` + +Each result is `median [p25, p75]`. The narrow hardened ranges contrast with the wider old DOM-pretty ranges on larger payloads. + +| Workload | Mode | Pre-hardening median [p25, p75] | Hardened median [p25, p75] | Change | +|----------|------|--------------------------------:|----------------------------:|-------:| +| Small | Default | 24.7µs [24.6, 25.0] | 6.2µs [6.2, 6.3] | **74.8% faster** | +| Small | Compact | 4.4µs [4.4, 4.4] | 6.4µs [6.3, 6.4] | **44.0% slower** | +| Small | Pretty | 24.7µs [24.5, 24.8] | 14.8µs [14.7, 14.9] | **40.0% faster** | +| 100 records | Default | 7.13ms [6.65, 8.50] | 1.75ms [1.74, 1.76] | **75.5% faster** | +| 100 records | Compact | 1.09ms [1.09, 1.11] | 1.75ms [1.74, 1.79] | **60.3% slower** | +| 100 records | Pretty | 6.85ms [6.61, 8.47] | 4.96ms [4.90, 5.03] | **27.5% faster** | +| 1,000 records | Default | 88.01ms [62.86, 100.53] | 17.24ms [17.17, 17.30] | **80.4% faster** | +| 1,000 records | Compact | 10.90ms [10.87, 10.98] | 17.21ms [17.15, 17.28] | **57.8% slower** | +| 1,000 records | Pretty | 88.83ms [63.04, 102.21] | 50.46ms [49.61, 51.15] | **43.2% faster** | + +#### Output Checks + +Compact output stayed byte-for-byte identical across revisions. Default and pretty output sizes differ because default now returns compact bytes and lexical pretty formatting replaces `minidom` formatting. + +| Workload | Compact bytes, both revisions | Default type/bytes, before → after | Pretty type/bytes, before → after | +|----------|------------------------------:|----------------------------------:|---------------------------------:| +| Small | 134 | `str`/142 → `bytes`/134 | `str`/142 → `str`/146 | +| 100 records | 57,049 | `str`/64,551 → `bytes`/57,049 | `str`/64,551 → `str`/69,952 | +| 1,000 records | 571,005 | `str`/646,007 → `bytes`/571,005 | `str`/646,007 → `str`/700,008 | #### Interpretation diff --git a/docs/benchmarks.rst b/docs/benchmarks.rst index 7eb927d0..7e4443bb 100644 --- a/docs/benchmarks.rst +++ b/docs/benchmarks.rst @@ -205,61 +205,98 @@ CPython 3.15.0rc1 Follow-up uv 0.12.3 downloaded and managed the exact ``cpython-3.15.0rc1-macos-aarch64-none`` build. The follow-up uses the same payloads, modes, warmups, sample counts, worker isolation, and ABBA ordering. +uv 0.11.24 did not yet list rc1, so it was updated to 0.12.3. That release downloaded the exact 26.2 MiB Apple Silicon build and verified its version before measurement. + +.. code-block:: bash + + uv self update + uv python list 3.15 --all-versions + uv run --isolated --managed-python \ + --python cpython-3.15.0rc1-macos-aarch64-none python --version + +Each result is ``median [p25, p75]``. The narrow hardened ranges contrast with the wider old DOM-pretty ranges on larger payloads. + .. list-table:: :header-rows: 1 - :widths: 24 18 20 20 18 + :widths: 17 14 27 27 15 * - Workload - Mode - - Pre-hardening - - Hardened master + - Pre-hardening median [p25, p75] + - Hardened median [p25, p75] - Change * - Small - Default - - 24.7µs - - 6.2µs + - 24.7µs [24.6, 25.0] + - 6.2µs [6.2, 6.3] - **74.8% faster** * - Small - Compact - - 4.4µs - - 6.4µs + - 4.4µs [4.4, 4.4] + - 6.4µs [6.3, 6.4] - **44.0% slower** * - Small - Pretty - - 24.7µs - - 14.8µs + - 24.7µs [24.5, 24.8] + - 14.8µs [14.7, 14.9] - **40.0% faster** * - 100 records - Default - - 7.13ms - - 1.75ms + - 7.13ms [6.65, 8.50] + - 1.75ms [1.74, 1.76] - **75.5% faster** * - 100 records - Compact - - 1.09ms - - 1.75ms + - 1.09ms [1.09, 1.11] + - 1.75ms [1.74, 1.79] - **60.3% slower** * - 100 records - Pretty - - 6.85ms - - 4.96ms + - 6.85ms [6.61, 8.47] + - 4.96ms [4.90, 5.03] - **27.5% faster** * - 1,000 records - Default - - 88.01ms - - 17.24ms + - 88.01ms [62.86, 100.53] + - 17.24ms [17.17, 17.30] - **80.4% faster** * - 1,000 records - Compact - - 10.90ms - - 17.21ms + - 10.90ms [10.87, 10.98] + - 17.21ms [17.15, 17.28] - **57.8% slower** * - 1,000 records - Pretty - - 88.83ms - - 50.46ms + - 88.83ms [63.04, 102.21] + - 50.46ms [49.61, 51.15] - **43.2% faster** +Output Checks +^^^^^^^^^^^^^ + +Compact output stayed byte-for-byte identical across revisions. Default and pretty sizes differ because default now returns compact bytes and lexical formatting replaces ``minidom`` formatting. + +.. list-table:: + :header-rows: 1 + :widths: 18 22 30 30 + + * - Workload + - Compact bytes, both + - Default type/bytes, before → after + - Pretty type/bytes, before → after + * - Small + - 134 + - ``str``/142 → ``bytes``/134 + - ``str``/142 → ``str``/146 + * - 100 records + - 57,049 + - ``str``/64,551 → ``bytes``/57,049 + - ``str``/64,551 → ``str``/69,952 + * - 1,000 records + - 571,005 + - ``str``/646,007 → ``bytes``/571,005 + - ``str``/646,007 → ``str``/700,008 + Default calls are roughly 4-5x faster because they now return compact serializer bytes. Explicit pretty output is 27-42% faster because lexical indentation replaces DOM parsing. Explicit compact output is 45-61% slower. Its bytes were identical and the serializer was unchanged, isolating the regression mainly to the new full-input resource-budget scan. diff --git a/lat.md/architecture.md b/lat.md/architecture.md index 79d4fc09..0a19bf35 100644 --- a/lat.md/architecture.md +++ b/lat.md/architecture.md @@ -64,6 +64,8 @@ Default calls improved 74-80% and pretty calls 27-42%, while explicit compact ca An identical uv-managed CPython 3.15.0rc1 follow-up confirmed the result: default calls improved 75-80%, pretty calls improved 28-43%, and compact calls regressed 44-60%. +The detailed record includes interquartile ranges, exact uv and interpreter provenance, and output checks showing that compact bytes remained identical across revisions. + The June 2026 Rust memory benchmark uses [[benchmark_memory_rust.py#main]] under hyperfine to compare release builds in fresh Python processes. The bytes-writer implementation cuts serializer peak RSS by about half for large outputs, with a documented throughput tradeoff. The June 2026 multi-interpreter CLI rerun uses [[benchmark_multi_python.py#main]] with per-interpreter virtual environments. On the recorded Apple Silicon run, CPython 3.15.0rc1 beat CPython 3.14.6 on every case, PyPy 3.11.15 only won the largest case, and Go remained the fastest end-to-end CLI path overall. From fa46cf64fd8997b41f57fa30650f2f44aa613d0c Mon Sep 17 00:00:00 2001 From: Vinit Kumar Date: Wed, 12 Aug 2026 23:43:11 +0530 Subject: [PATCH 4/5] bench: make security measurements reproducible --- BENCHMARKS.md | 33 +- benchmark_security_hardening.py | 378 +++++++++++++++++++++ docs/benchmarks.rst | 27 +- lat.md/architecture.md | 4 +- lat.md/tests.md | 12 + tests/test_benchmark_security_hardening.py | 57 ++++ 6 files changed, 503 insertions(+), 8 deletions(-) create mode 100644 benchmark_security_hardening.py create mode 100644 tests/test_benchmark_security_hardening.py diff --git a/BENCHMARKS.md b/BENCHMARKS.md index 60e7552b..00b5bf72 100644 --- a/BENCHMARKS.md +++ b/BENCHMARKS.md @@ -101,10 +101,14 @@ It compares pre-hardening commit `826439f` with hardened `master` at `48dfd38`. - Apple Silicon and macOS 26.6.1, with CPython 3.14.6 for the primary run. - CPython 3.15.0rc1 follow-up downloaded and managed by uv 0.12.3. -- Deterministic small, 100-record, and 1,000-record nested payloads. +- Deterministic small, 100-record, and 1,000-record nested payloads generated by `benchmark_security_hardening.py`. - Default, explicit `pretty=False`, and explicit `pretty=True` calls. - Five warmups per worker and 68 timed samples per cell across four fresh workers per revision. -- Revisions were interleaved in ABBA order to reduce thermal and scheduler bias. +- Revisions were interleaved in mirrored ABBA and BAAB orders to give each revision every worker position. + +The payload generator is deterministic by construction: record fields are derived only from each record's zero-based index, so there is no pseudorandom generator or external seed to supply. The committed harness contains the exact payloads, loop counts, worker order, timing code, and UTF-8 output checks. + +It resolves both revisions to full commit IDs, creates detached temporary worktrees, and launches a fresh Python process for every worker. Each ABBA/BAAB pass contributes two workers per revision; two passes × 17 samples produce the documented 68 samples per revision and workload/mode cell. #### CPython Results @@ -124,7 +128,7 @@ Each timing is the median public-API conversion time. “Faster” and “slower #### CPython 3.15.0rc1 Results -The rc1 follow-up uses the same payloads, modes, warmups, sample counts, worker isolation, and ABBA revision ordering as the CPython 3.14.6 run. +The rc1 follow-up uses the same payloads, modes, warmups, sample counts, worker isolation, and mirrored ABBA/BAAB revision ordering as the CPython 3.14.6 run. uv 0.11.24 did not yet list rc1, so it was updated to 0.12.3. That release downloaded the exact 26.2 MiB Apple Silicon build and verified its version before measurement. @@ -132,9 +136,17 @@ uv 0.11.24 did not yet list rc1, so it was updated to 0.12.3. That release downl uv self update uv python list 3.15 --all-versions uv run --isolated --managed-python \ - --python cpython-3.15.0rc1-macos-aarch64-none python --version + --python cpython-3.15.0rc1-macos-aarch64-none \ + --with defusedxml \ + --with 'urllib3>=2.7.0' \ + python benchmark_security_hardening.py \ + --before 826439f \ + --after 48dfd38 \ + --output-json /tmp/json2xml-security-benchmark-python315rc1.json ``` +The console output contains the median and interquartile range plus output type, UTF-8 byte count, and full SHA-256 for every cell. The optional JSON file preserves all 1,224 raw timings: 9 cells × 2 revisions × 68 samples. + Each result is `median [p25, p75]`. The narrow hardened ranges contrast with the wider old DOM-pretty ranges on larger payloads. | Workload | Mode | Pre-hardening median [p25, p75] | Hardened median [p25, p75] | Change | @@ -159,6 +171,8 @@ Compact output stayed byte-for-byte identical across revisions. Default and pret | 100 records | 57,049 | `str`/64,551 → `bytes`/57,049 | `str`/64,551 → `str`/69,952 | | 1,000 records | 571,005 | `str`/646,007 → `bytes`/571,005 | `str`/646,007 → `str`/700,008 | +The compact-output SHA-256 values are `013c28cd992bb5caa9a3c357a492ab47ccf00ad479ad3010e7ec376754cb06a8` (small), `05d79278d394c70a816fce23aa60e2fd0059233abf94c863a26e17af4dcdb090` (100 records), and `72de1be56cbc381c25606a3ca5c73b40d482ee08ef2d74f93d3b2798e531d760` (1,000 records). + #### Interpretation Default calls are roughly 4-5x faster because they now return compact serializer bytes instead of building pretty output. This comparison includes the intentional default-output contract change. @@ -315,6 +329,17 @@ and PyPy 3.11.15 under `JSON2XML_UV_PYTHON_DIR` (default: python benchmark_multi_python.py ``` +### Security Hardening Public-API Benchmark + +Compares two commits through `Json2xml(...).to_xml()` in default, compact, and pretty modes. The script creates and removes detached temporary worktrees automatically; it does not alter the current checkout. + +```bash +python benchmark_security_hardening.py \ + --before 826439f \ + --after 48dfd38 \ + --output-json /tmp/json2xml-security-benchmark.json +``` + ### Interpreting CLI Numbers The Go and Zig rows measure full process startup plus conversion because diff --git a/benchmark_security_hardening.py b/benchmark_security_hardening.py new file mode 100644 index 00000000..8103b046 --- /dev/null +++ b/benchmark_security_hardening.py @@ -0,0 +1,378 @@ +#!/usr/bin/env python3 +"""Reproduce the public-wrapper benchmark for the security-hardening change.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import platform +import statistics +import subprocess +import sys +import tempfile +import time +from pathlib import Path +from typing import Any + +BEFORE = "before" +AFTER = "after" +DEFAULT_BEFORE_REVISION = "826439f" +DEFAULT_AFTER_REVISION = "48dfd38" +SAMPLES_PER_WORKER = 17 +WARMUPS_PER_WORKER = 5 +WORKERS_PER_REVISION = 4 + +# The mirrored order gives each revision every position in the four-process sequence. +WORKER_ORDERS = ( + (BEFORE, AFTER, AFTER, BEFORE), + (AFTER, BEFORE, BEFORE, AFTER), +) + +# (display name, number of records, conversions per timed sample) +CASES = ( + ("Small", 0, 1000), + ("100 records", 100, 10), + ("1,000 records", 1000, 1), +) +MODES = ("default", "compact", "pretty") + + +def make_payload(records: int) -> dict[str, Any] | list[dict[str, Any]]: + """Return the exact deterministic payload used by every benchmark worker.""" + if records == 0: + return {"name": "John", "age": 30, "city": "New York"} + + return [ + { + "id": index, + "name": f"customer-{index:08d}-" + "name" * 5, + "email": f"user-{index:08d}@example.com", + "active": index % 2 == 0, + "score": (index % 10000) / 17.0, + "tags": [f"tag-{index % 17}", f"region-{index % 23}", "xml-safe"], + "metadata": { + "created": "2026-08-12T10:30:00Z", + "version": index % 101, + "nested": {"level1": {"value": f"value-{index:08d}"}}, + }, + } + for index in range(records) + ] + + +def _mode_kwargs(mode: str) -> dict[str, Any]: + if mode == "default": + return {} + if mode == "compact": + return {"pretty": False} + if mode == "pretty": + return {"pretty": True} + raise ValueError(f"unknown benchmark mode: {mode}") + + +def _worker(source: Path, mode: str, records: int, loops: int) -> None: + os.chdir(source) + sys.path.insert(0, str(source)) + + from json2xml.json2xml import Json2xml + + payload = make_payload(records) + kwargs = _mode_kwargs(mode) + + def convert() -> bytes | str | None: + return Json2xml(payload, **kwargs).to_xml() + + for _ in range(WARMUPS_PER_WORKER): + for _ in range(loops): + convert() + + samples_ns: list[float] = [] + for _ in range(SAMPLES_PER_WORKER): + started_ns = time.perf_counter_ns() + for _ in range(loops): + result = convert() + samples_ns.append((time.perf_counter_ns() - started_ns) / loops) + + if result is None: + raise RuntimeError("benchmark conversion unexpectedly returned None") + encoded = result if isinstance(result, bytes) else result.encode("utf-8") + print( + json.dumps( + { + "samples_ns": samples_ns, + "output": { + "type": type(result).__name__, + "utf8_bytes": len(encoded), + "sha256": hashlib.sha256(encoded).hexdigest(), + }, + }, + sort_keys=True, + ) + ) + + +def _run(command: list[str], cwd: Path) -> subprocess.CompletedProcess[str]: + return subprocess.run( + command, + cwd=cwd, + check=True, + capture_output=True, + text=True, + ) + + +def _resolve_revision(repo: Path, revision: str) -> str: + result = _run( + [ + "git", + "rev-parse", + "--verify", + "--end-of-options", + f"{revision}^{{commit}}", + ], + repo, + ) + return result.stdout.strip() + + +def _invoke_worker( + script: Path, + repo: Path, + source: Path, + mode: str, + records: int, + loops: int, +) -> dict[str, Any]: + result = _run( + [ + sys.executable, + str(script), + "--worker", + "--source", + str(source), + "--mode", + mode, + "--records", + str(records), + "--loops", + str(loops), + ], + repo, + ) + return json.loads(result.stdout) + + +def _percentile(values: list[float], percentile: float) -> float: + ordered = sorted(values) + position = (len(ordered) - 1) * percentile + lower = int(position) + upper = min(lower + 1, len(ordered) - 1) + fraction = position - lower + return ordered[lower] + (ordered[upper] - ordered[lower]) * fraction + + +def _summarize(samples_ns: list[float], output: dict[str, Any]) -> dict[str, Any]: + return { + "samples_ns": samples_ns, + "median_ns": statistics.median(samples_ns), + "p25_ns": _percentile(samples_ns, 0.25), + "p75_ns": _percentile(samples_ns, 0.75), + "output": output, + } + + +def _benchmark_cell( + script: Path, + repo: Path, + sources: dict[str, Path], + workload: str, + records: int, + loops: int, + mode: str, +) -> dict[str, Any]: + samples: dict[str, list[float]] = {BEFORE: [], AFTER: []} + outputs: dict[str, dict[str, Any]] = {} + + for order in WORKER_ORDERS: + for label in order: + worker_result = _invoke_worker( + script, repo, sources[label], mode, records, loops + ) + samples[label].extend(worker_result["samples_ns"]) + output = worker_result["output"] + if label in outputs and output != outputs[label]: + raise RuntimeError(f"{label} workers produced different output") + outputs[label] = output + + before = _summarize(samples[BEFORE], outputs[BEFORE]) + after = _summarize(samples[AFTER], outputs[AFTER]) + if len(samples[BEFORE]) != 68 or len(samples[AFTER]) != 68: + raise RuntimeError("benchmark schedule did not produce 68 samples per revision") + + change_percent = ( + (after["median_ns"] - before["median_ns"]) / before["median_ns"] * 100 + ) + return { + "workload": workload, + "records": records, + "loops_per_sample": loops, + "mode": mode, + "workers_per_revision": WORKERS_PER_REVISION, + "warmups_per_worker": WARMUPS_PER_WORKER, + "samples_per_worker": SAMPLES_PER_WORKER, + "before": before, + "after": after, + "change_percent": change_percent, + "outputs_identical": outputs[BEFORE] == outputs[AFTER], + } + + +def _format_duration(nanoseconds: float) -> str: + if nanoseconds < 1000: + return f"{nanoseconds:.1f}ns" + if nanoseconds < 1_000_000: + return f"{nanoseconds / 1000:.1f}us" + return f"{nanoseconds / 1_000_000:.2f}ms" + + +def _print_report(report: dict[str, Any]) -> None: + print(f"Python: {report['environment']['python'].splitlines()[0]}") + print(f"Executable: {report['environment']['executable']}") + print(f"Platform: {report['environment']['platform']}") + print( + "Revisions: " + f"{report['revisions']['before']} (before) -> " + f"{report['revisions']['after']} (after)" + ) + print() + print("| Workload | Mode | Before median [p25, p75] | After median [p25, p75] | Change |") + print("|---|---|---:|---:|---:|") + for cell in report["cells"]: + before = cell["before"] + after = cell["after"] + print( + f"| {cell['workload']} | {cell['mode']} | " + f"{_format_duration(before['median_ns'])} " + f"[{_format_duration(before['p25_ns'])}, {_format_duration(before['p75_ns'])}] | " + f"{_format_duration(after['median_ns'])} " + f"[{_format_duration(after['p25_ns'])}, {_format_duration(after['p75_ns'])}] | " + f"{cell['change_percent']:+.1f}% |" + ) + print() + print("| Workload | Mode | Before type/bytes/SHA-256 | After type/bytes/SHA-256 | Identical |") + print("|---|---|---|---|---:|") + for cell in report["cells"]: + before = cell["before"]["output"] + after = cell["after"]["output"] + print( + f"| {cell['workload']} | {cell['mode']} | " + f"{before['type']}/{before['utf8_bytes']}/{before['sha256']} | " + f"{after['type']}/{after['utf8_bytes']}/{after['sha256']} | " + f"{str(cell['outputs_identical']).lower()} |" + ) + + +def run_benchmark(before_revision: str, after_revision: str) -> dict[str, Any]: + script = Path(__file__).resolve() + repo = Path( + _run(["git", "rev-parse", "--show-toplevel"], script.parent).stdout.strip() + ) + revisions = { + BEFORE: _resolve_revision(repo, before_revision), + AFTER: _resolve_revision(repo, after_revision), + } + + with tempfile.TemporaryDirectory(prefix="json2xml-security-benchmark-") as temp: + temp_path = Path(temp) + sources = {BEFORE: temp_path / BEFORE, AFTER: temp_path / AFTER} + added: list[Path] = [] + try: + for label in (BEFORE, AFTER): + _run( + [ + "git", + "worktree", + "add", + "--detach", + str(sources[label]), + revisions[label], + ], + repo, + ) + added.append(sources[label]) + + cells = [ + _benchmark_cell( + script, repo, sources, workload, records, loops, mode + ) + for workload, records, loops in CASES + for mode in MODES + ] + finally: + for source in reversed(added): + subprocess.run( + ["git", "worktree", "remove", "--force", str(source)], + cwd=repo, + check=False, + capture_output=True, + text=True, + ) + + return { + "schema_version": 1, + "environment": { + "python": sys.version, + "executable": sys.executable, + "platform": platform.platform(), + }, + "revisions": { + "before": revisions[BEFORE], + "after": revisions[AFTER], + }, + "schedule": { + "orders": WORKER_ORDERS, + "workers_per_revision": WORKERS_PER_REVISION, + "warmups_per_worker": WARMUPS_PER_WORKER, + "samples_per_worker": SAMPLES_PER_WORKER, + "samples_per_revision_and_cell": WORKERS_PER_REVISION + * SAMPLES_PER_WORKER, + }, + "cells": cells, + } + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--before", default=DEFAULT_BEFORE_REVISION) + parser.add_argument("--after", default=DEFAULT_AFTER_REVISION) + parser.add_argument("--output-json", type=Path) + parser.add_argument("--worker", action="store_true", help=argparse.SUPPRESS) + parser.add_argument("--source", type=Path, help=argparse.SUPPRESS) + parser.add_argument("--mode", choices=MODES, help=argparse.SUPPRESS) + parser.add_argument("--records", type=int, help=argparse.SUPPRESS) + parser.add_argument("--loops", type=int, help=argparse.SUPPRESS) + return parser + + +# @lat: [[architecture#Performance benchmarks]] +def main() -> None: + args = _parser().parse_args() + if args.worker: + if args.source is None or args.mode is None or args.records is None or args.loops is None: + raise SystemExit("worker mode requires source, mode, records, and loops") + _worker(args.source, args.mode, args.records, args.loops) + return + + report = run_benchmark(args.before, args.after) + _print_report(report) + if args.output_json is not None: + args.output_json.write_text( + json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + print(f"\nRaw samples written to {args.output_json}") + + +if __name__ == "__main__": + main() diff --git a/docs/benchmarks.rst b/docs/benchmarks.rst index 7e4443bb..42c7b3f1 100644 --- a/docs/benchmarks.rst +++ b/docs/benchmarks.rst @@ -143,7 +143,9 @@ This August 12, 2026 comparison measures the public ``Json2xml(...).to_xml()`` p It compares pre-hardening commit ``826439f`` with hardened ``master`` at ``48dfd38`` on Apple Silicon and macOS 26.6.1. The primary run used CPython 3.14.6. -The deterministic benchmark used small, 100-record, and 1,000-record nested payloads. Each cell reports the median of 68 timed samples across four fresh workers per revision, interleaved in ABBA order after five warmups. +The deterministic benchmark used small, 100-record, and 1,000-record nested payloads generated by ``benchmark_security_hardening.py``. Record fields depend only on their zero-based index, so no pseudorandom seed or hidden input is required. + +The committed harness contains the exact payloads, loop counts, worker order, timing code, and UTF-8 output checks. It creates detached temporary worktrees for both revisions and launches fresh workers in mirrored ABBA and BAAB orders. Two passes provide four workers per revision; 17 samples per worker yield 68 samples per revision and cell after five warmups per worker. .. list-table:: :header-rows: 1 @@ -203,7 +205,7 @@ The deterministic benchmark used small, 100-record, and 1,000-record nested payl CPython 3.15.0rc1 Follow-up ^^^^^^^^^^^^^^^^^^^^^^^^^^^ -uv 0.12.3 downloaded and managed the exact ``cpython-3.15.0rc1-macos-aarch64-none`` build. The follow-up uses the same payloads, modes, warmups, sample counts, worker isolation, and ABBA ordering. +uv 0.12.3 downloaded and managed the exact ``cpython-3.15.0rc1-macos-aarch64-none`` build. The follow-up uses the same payloads, modes, warmups, sample counts, worker isolation, and mirrored ABBA/BAAB ordering. uv 0.11.24 did not yet list rc1, so it was updated to 0.12.3. That release downloaded the exact 26.2 MiB Apple Silicon build and verified its version before measurement. @@ -212,7 +214,15 @@ uv 0.11.24 did not yet list rc1, so it was updated to 0.12.3. That release downl uv self update uv python list 3.15 --all-versions uv run --isolated --managed-python \ - --python cpython-3.15.0rc1-macos-aarch64-none python --version + --python cpython-3.15.0rc1-macos-aarch64-none \ + --with defusedxml \ + --with 'urllib3>=2.7.0' \ + python benchmark_security_hardening.py \ + --before 826439f \ + --after 48dfd38 \ + --output-json /tmp/json2xml-security-benchmark-python315rc1.json + +The console report includes medians, interquartile ranges, output types, UTF-8 byte counts, and SHA-256 values. The optional JSON file preserves all 1,224 raw timings: 9 cells × 2 revisions × 68 samples. Each result is ``median [p25, p75]``. The narrow hardened ranges contrast with the wider old DOM-pretty ranges on larger payloads. @@ -297,6 +307,8 @@ Compact output stayed byte-for-byte identical across revisions. Default and pret - ``str``/646,007 → ``bytes``/571,005 - ``str``/646,007 → ``str``/700,008 +The compact-output SHA-256 values are ``013c28cd992bb5caa9a3c357a492ab47ccf00ad479ad3010e7ec376754cb06a8`` for small, ``05d79278d394c70a816fce23aa60e2fd0059233abf94c863a26e17af4dcdb090`` for 100 records, and ``72de1be56cbc381c25606a3ca5c73b40d482ee08ef2d74f93d3b2798e531d760`` for 1,000 records. + Default calls are roughly 4-5x faster because they now return compact serializer bytes. Explicit pretty output is 27-42% faster because lexical indentation replaces DOM parsing. Explicit compact output is 45-61% slower. Its bytes were identical and the serializer was unchanged, isolating the regression mainly to the new full-input resource-budget scan. @@ -354,6 +366,15 @@ Run benchmarks from a clean checkout with the project installed in an isolated e uv pip install -e . python benchmark_all.py +To reproduce the security-hardening public-API comparison on the active Python interpreter, run the committed harness. It removes its detached temporary worktrees without changing the current checkout. + +.. code-block:: bash + + python benchmark_security_hardening.py \ + --before 826439f \ + --after 48dfd38 \ + --output-json /tmp/json2xml-security-benchmark.json + For Rust benchmarks, install the extension into the same environment. .. code-block:: bash diff --git a/lat.md/architecture.md b/lat.md/architecture.md index 0a19bf35..8f180707 100644 --- a/lat.md/architecture.md +++ b/lat.md/architecture.md @@ -58,7 +58,9 @@ The May 2026 benchmark on Apple Silicon shows the Rust extension as the best opt Reproduction docs require contributors to record machine, OS, Python, and tool availability before comparing results. `benchmark_all.py` mixes library calls and CLI subprocesses intentionally, so its Go and Zig rows include process startup overhead. -The August 2026 public-wrapper benchmark compares pre-hardening `826439f` with hardened `48dfd38` using ABBA-interleaved CPython 3.14 samples. +The August 2026 public-wrapper benchmark uses [[benchmark_security_hardening.py#main]] to compare pre-hardening `826439f` with hardened `48dfd38` using fresh ABBA/BAAB-interleaved workers. + +Its [[benchmark_security_hardening.py#make_payload]] generator derives every field from a record index, while four workers × 17 samples give each revision 68 observations per cell. Raw JSON includes every timing plus output type, byte count, and SHA-256. Default calls improved 74-80% and pretty calls 27-42%, while explicit compact calls regressed 45-61% from the resource-budget scan. diff --git a/lat.md/tests.md b/lat.md/tests.md index 8fac266d..45957dfa 100644 --- a/lat.md/tests.md +++ b/lat.md/tests.md @@ -78,6 +78,18 @@ The multi-interpreter benchmark should derive default interpreter paths from `JS The multi-interpreter benchmark should let per-interpreter environment variables override uv-derived defaults so unusual local layouts remain runnable without editing the script. +### Security benchmark payloads stay deterministic + +The public-wrapper benchmark should build the documented small and index-derived nested record payloads exactly so reruns use identical inputs without an external random seed. + +### Security benchmark schedule yields 68 samples + +The public-wrapper benchmark should run mirrored ABBA and BAAB worker orders with 17 samples per worker, yielding 68 balanced observations for each revision and cell. + +### Security benchmark covers every published cell + +The public-wrapper benchmark should retain the documented small, 100-record, and 1,000-record loop counts in default, compact, and pretty modes. + ## Conversion behavior These tests pin the XML shapes that matter most for interoperability, especially the modes that intentionally diverge from the default serializer. diff --git a/tests/test_benchmark_security_hardening.py b/tests/test_benchmark_security_hardening.py new file mode 100644 index 00000000..1abc403b --- /dev/null +++ b/tests/test_benchmark_security_hardening.py @@ -0,0 +1,57 @@ +from __future__ import annotations + +from collections import Counter + +import benchmark_security_hardening as benchmark + + +# @lat: [[tests#Performance benchmarks#Security benchmark payloads stay deterministic]] +def test_security_benchmark_payloads_stay_deterministic() -> None: + assert benchmark.make_payload(0) == { + "name": "John", + "age": 30, + "city": "New York", + } + + payload = benchmark.make_payload(2) + + assert isinstance(payload, list) + assert payload == benchmark.make_payload(2) + assert payload[1] == { + "id": 1, + "name": "customer-00000001-namenamenamenamename", + "email": "user-00000001@example.com", + "active": False, + "score": 1 / 17.0, + "tags": ["tag-1", "region-1", "xml-safe"], + "metadata": { + "created": "2026-08-12T10:30:00Z", + "version": 1, + "nested": {"level1": {"value": "value-00000001"}}, + }, + } + + +# @lat: [[tests#Performance benchmarks#Security benchmark schedule yields 68 samples]] +def test_security_benchmark_schedule_yields_68_balanced_samples() -> None: + flattened_orders = [label for order in benchmark.WORKER_ORDERS for label in order] + + assert benchmark.WORKER_ORDERS == ( + (benchmark.BEFORE, benchmark.AFTER, benchmark.AFTER, benchmark.BEFORE), + (benchmark.AFTER, benchmark.BEFORE, benchmark.BEFORE, benchmark.AFTER), + ) + assert Counter(flattened_orders) == {benchmark.BEFORE: 4, benchmark.AFTER: 4} + assert benchmark.WORKERS_PER_REVISION * benchmark.SAMPLES_PER_WORKER == 68 + + +# @lat: [[tests#Performance benchmarks#Security benchmark covers every published cell]] +def test_security_benchmark_covers_every_published_cell() -> None: + assert benchmark.CASES == ( + ("Small", 0, 1000), + ("100 records", 100, 10), + ("1,000 records", 1000, 1), + ) + assert benchmark.MODES == ("default", "compact", "pretty") + assert benchmark._mode_kwargs("default") == {} + assert benchmark._mode_kwargs("compact") == {"pretty": False} + assert benchmark._mode_kwargs("pretty") == {"pretty": True} From 93e61eeda5278263b09781986eb68e5409b7080b Mon Sep 17 00:00:00 2001 From: Vinit Kumar Date: Wed, 12 Aug 2026 23:48:09 +0530 Subject: [PATCH 5/5] fix: make benchmark subprocess boundaries auditable --- BENCHMARKS.md | 2 + benchmark_security_hardening.py | 45 +++++++++++++--------- docs/benchmarks.rst | 2 + lat.md/architecture.md | 2 + lat.md/tests.md | 4 ++ tests/test_benchmark_security_hardening.py | 27 +++++++++++++ 6 files changed, 64 insertions(+), 18 deletions(-) diff --git a/BENCHMARKS.md b/BENCHMARKS.md index 00b5bf72..7f584246 100644 --- a/BENCHMARKS.md +++ b/BENCHMARKS.md @@ -110,6 +110,8 @@ The payload generator is deterministic by construction: record fields are derive It resolves both revisions to full commit IDs, creates detached temporary worktrees, and launches a fresh Python process for every worker. Each ABBA/BAAB pass contributes two workers per revision; two passes × 17 samples produce the documented 68 samples per revision and workload/mode cell. +Every child process receives an argv list with shell parsing disabled. User-supplied revision text is placed after Git's `--end-of-options` marker, and only the resulting full commit ID is passed to `git worktree add`; shell escaping is neither needed nor used. + #### CPython Results Each timing is the median public-API conversion time. “Faster” and “slower” compare hardened `master` with the pre-hardening revision. diff --git a/benchmark_security_hardening.py b/benchmark_security_hardening.py index 8103b046..1f106d98 100644 --- a/benchmark_security_hardening.py +++ b/benchmark_security_hardening.py @@ -113,18 +113,8 @@ def convert() -> bytes | str | None: ) -def _run(command: list[str], cwd: Path) -> subprocess.CompletedProcess[str]: - return subprocess.run( - command, - cwd=cwd, - check=True, - capture_output=True, - text=True, - ) - - def _resolve_revision(repo: Path, revision: str) -> str: - result = _run( + result = subprocess.run( [ "git", "rev-parse", @@ -132,7 +122,11 @@ def _resolve_revision(repo: Path, revision: str) -> str: "--end-of-options", f"{revision}^{{commit}}", ], - repo, + cwd=repo, + check=True, + capture_output=True, + text=True, + shell=False, ) return result.stdout.strip() @@ -145,7 +139,7 @@ def _invoke_worker( records: int, loops: int, ) -> dict[str, Any]: - result = _run( + result = subprocess.run( [ sys.executable, str(script), @@ -159,7 +153,11 @@ def _invoke_worker( "--loops", str(loops), ], - repo, + cwd=repo, + check=True, + capture_output=True, + text=True, + shell=False, ) return json.loads(result.stdout) @@ -276,9 +274,15 @@ def _print_report(report: dict[str, Any]) -> None: def run_benchmark(before_revision: str, after_revision: str) -> dict[str, Any]: script = Path(__file__).resolve() - repo = Path( - _run(["git", "rev-parse", "--show-toplevel"], script.parent).stdout.strip() + repo_result = subprocess.run( + ["git", "rev-parse", "--show-toplevel"], + cwd=script.parent, + check=True, + capture_output=True, + text=True, + shell=False, ) + repo = Path(repo_result.stdout.strip()) revisions = { BEFORE: _resolve_revision(repo, before_revision), AFTER: _resolve_revision(repo, after_revision), @@ -290,7 +294,7 @@ def run_benchmark(before_revision: str, after_revision: str) -> dict[str, Any]: added: list[Path] = [] try: for label in (BEFORE, AFTER): - _run( + subprocess.run( [ "git", "worktree", @@ -299,7 +303,11 @@ def run_benchmark(before_revision: str, after_revision: str) -> dict[str, Any]: str(sources[label]), revisions[label], ], - repo, + cwd=repo, + check=True, + capture_output=True, + text=True, + shell=False, ) added.append(sources[label]) @@ -318,6 +326,7 @@ def run_benchmark(before_revision: str, after_revision: str) -> dict[str, Any]: check=False, capture_output=True, text=True, + shell=False, ) return { diff --git a/docs/benchmarks.rst b/docs/benchmarks.rst index 42c7b3f1..b86e8aa4 100644 --- a/docs/benchmarks.rst +++ b/docs/benchmarks.rst @@ -147,6 +147,8 @@ The deterministic benchmark used small, 100-record, and 1,000-record nested payl The committed harness contains the exact payloads, loop counts, worker order, timing code, and UTF-8 output checks. It creates detached temporary worktrees for both revisions and launches fresh workers in mirrored ABBA and BAAB orders. Two passes provide four workers per revision; 17 samples per worker yield 68 samples per revision and cell after five warmups per worker. +Every child process receives an argv list with shell parsing disabled. User-supplied revision text follows Git's ``--end-of-options`` marker, and only the resulting full commit ID is passed to ``git worktree add``; no shell escaping is required. + .. list-table:: :header-rows: 1 :widths: 24 18 20 20 18 diff --git a/lat.md/architecture.md b/lat.md/architecture.md index 8f180707..6ccfc00e 100644 --- a/lat.md/architecture.md +++ b/lat.md/architecture.md @@ -62,6 +62,8 @@ The August 2026 public-wrapper benchmark uses [[benchmark_security_hardening.py# Its [[benchmark_security_hardening.py#make_payload]] generator derives every field from a record index, while four workers × 17 samples give each revision 68 observations per cell. Raw JSON includes every timing plus output type, byte count, and SHA-256. +Harness subprocesses use inline argv lists with shell parsing disabled. Revision arguments follow Git's end-of-options marker, and worktrees receive only full commit IDs resolved by Git. + Default calls improved 74-80% and pretty calls 27-42%, while explicit compact calls regressed 45-61% from the resource-budget scan. An identical uv-managed CPython 3.15.0rc1 follow-up confirmed the result: default calls improved 75-80%, pretty calls improved 28-43%, and compact calls regressed 44-60%. diff --git a/lat.md/tests.md b/lat.md/tests.md index 45957dfa..be438240 100644 --- a/lat.md/tests.md +++ b/lat.md/tests.md @@ -90,6 +90,10 @@ The public-wrapper benchmark should run mirrored ABBA and BAAB worker orders wit The public-wrapper benchmark should retain the documented small, 100-record, and 1,000-record loop counts in default, compact, and pretty modes. +### Security benchmark subprocess commands stay structured + +Every harness subprocess should use an inline argv list with shell parsing explicitly disabled so dynamic values cannot become command syntax and static security audits can verify the boundary. + ## Conversion behavior These tests pin the XML shapes that matter most for interoperability, especially the modes that intentionally diverge from the default serializer. diff --git a/tests/test_benchmark_security_hardening.py b/tests/test_benchmark_security_hardening.py index 1abc403b..8b655f10 100644 --- a/tests/test_benchmark_security_hardening.py +++ b/tests/test_benchmark_security_hardening.py @@ -1,6 +1,8 @@ from __future__ import annotations +import ast from collections import Counter +from pathlib import Path import benchmark_security_hardening as benchmark @@ -55,3 +57,28 @@ def test_security_benchmark_covers_every_published_cell() -> None: assert benchmark._mode_kwargs("default") == {} assert benchmark._mode_kwargs("compact") == {"pretty": False} assert benchmark._mode_kwargs("pretty") == {"pretty": True} + + +# @lat: [[tests#Performance benchmarks#Security benchmark subprocess commands stay structured]] +def test_security_benchmark_subprocess_commands_stay_structured() -> None: + source = Path(benchmark.__file__).read_text(encoding="utf-8") + tree = ast.parse(source) + subprocess_calls = [ + node + for node in ast.walk(tree) + if isinstance(node, ast.Call) + and isinstance(node.func, ast.Attribute) + and isinstance(node.func.value, ast.Name) + and node.func.value.id == "subprocess" + and node.func.attr == "run" + ] + + assert subprocess_calls + for call in subprocess_calls: + assert call.args and isinstance(call.args[0], ast.List) + shell_keyword = next( + (keyword for keyword in call.keywords if keyword.arg == "shell"), None + ) + assert shell_keyword is not None + assert isinstance(shell_keyword.value, ast.Constant) + assert shell_keyword.value.value is False